1//===- OpenMPIRBuilder.cpp - Builder for LLVM-IR for OpenMP directives ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9///
10/// This file implements the OpenMPIRBuilder class, which is used as a
11/// convenient way to create LLVM instructions for OpenMP directives.
12///
13//===----------------------------------------------------------------------===//
14
15#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
16#include "llvm/ADT/SmallBitVector.h"
17#include "llvm/ADT/SmallSet.h"
18#include "llvm/ADT/SmallVectorExtras.h"
19#include "llvm/ADT/StringExtras.h"
20#include "llvm/ADT/StringRef.h"
21#include "llvm/Analysis/AssumptionCache.h"
22#include "llvm/Analysis/CodeMetrics.h"
23#include "llvm/Analysis/LoopInfo.h"
24#include "llvm/Analysis/OptimizationRemarkEmitter.h"
25#include "llvm/Analysis/PostDominators.h"
26#include "llvm/Analysis/ScalarEvolution.h"
27#include "llvm/Analysis/TargetLibraryInfo.h"
28#include "llvm/Bitcode/BitcodeReader.h"
29#include "llvm/Frontend/Offloading/Utility.h"
30#include "llvm/Frontend/OpenMP/OMPGridValues.h"
31#include "llvm/IR/Attributes.h"
32#include "llvm/IR/BasicBlock.h"
33#include "llvm/IR/CFG.h"
34#include "llvm/IR/CallingConv.h"
35#include "llvm/IR/Constant.h"
36#include "llvm/IR/Constants.h"
37#include "llvm/IR/DIBuilder.h"
38#include "llvm/IR/DebugInfoMetadata.h"
39#include "llvm/IR/DerivedTypes.h"
40#include "llvm/IR/Function.h"
41#include "llvm/IR/GlobalVariable.h"
42#include "llvm/IR/IRBuilder.h"
43#include "llvm/IR/InstIterator.h"
44#include "llvm/IR/IntrinsicInst.h"
45#include "llvm/IR/LLVMContext.h"
46#include "llvm/IR/MDBuilder.h"
47#include "llvm/IR/Metadata.h"
48#include "llvm/IR/PassInstrumentation.h"
49#include "llvm/IR/PassManager.h"
50#include "llvm/IR/ReplaceConstant.h"
51#include "llvm/IR/Value.h"
52#include "llvm/MC/TargetRegistry.h"
53#include "llvm/Support/CommandLine.h"
54#include "llvm/Support/Error.h"
55#include "llvm/Support/ErrorHandling.h"
56#include "llvm/Support/FileSystem.h"
57#include "llvm/Support/NVVMAttributes.h"
58#include "llvm/Support/VirtualFileSystem.h"
59#include "llvm/Target/TargetMachine.h"
60#include "llvm/Target/TargetOptions.h"
61#include "llvm/Transforms/Utils/BasicBlockUtils.h"
62#include "llvm/Transforms/Utils/Cloning.h"
63#include "llvm/Transforms/Utils/CodeExtractor.h"
64#include "llvm/Transforms/Utils/LoopPeel.h"
65#include "llvm/Transforms/Utils/UnrollLoop.h"
66
67#include <cstdint>
68#include <optional>
69
70#define DEBUG_TYPE "openmp-ir-builder"
71
72using namespace llvm;
73using namespace omp;
74
75static cl::opt<bool>
76 OptimisticAttributes("openmp-ir-builder-optimistic-attributes", cl::Hidden,
77 cl::desc("Use optimistic attributes describing "
78 "'as-if' properties of runtime calls."),
79 cl::init(Val: false));
80
81static cl::opt<double> UnrollThresholdFactor(
82 "openmp-ir-builder-unroll-threshold-factor", cl::Hidden,
83 cl::desc("Factor for the unroll threshold to account for code "
84 "simplifications still taking place"),
85 cl::init(Val: 1.5));
86
87static cl::opt<bool> UseDefaultMaxThreads(
88 "openmp-ir-builder-use-default-max-threads", cl::Hidden,
89 cl::desc("Use a default max threads if none is provided."), cl::init(Val: true));
90
91#ifndef NDEBUG
92/// Return whether IP1 and IP2 are ambiguous, i.e. that inserting instructions
93/// at position IP1 may change the meaning of IP2 or vice-versa. This is because
94/// an InsertPoint stores the instruction before something is inserted. For
95/// instance, if both point to the same instruction, two IRBuilders alternating
96/// creating instruction will cause the instructions to be interleaved.
97static bool isConflictIP(IRBuilder<>::InsertPoint IP1,
98 IRBuilder<>::InsertPoint IP2) {
99 if (!IP1.isSet() || !IP2.isSet())
100 return false;
101 return IP1.getBlock() == IP2.getBlock() && IP1.getPoint() == IP2.getPoint();
102}
103
104static bool isValidWorkshareLoopScheduleType(OMPScheduleType SchedType) {
105 // Valid ordered/unordered and base algorithm combinations.
106 switch (SchedType & ~OMPScheduleType::MonotonicityMask) {
107 case OMPScheduleType::UnorderedStaticChunked:
108 case OMPScheduleType::UnorderedStatic:
109 case OMPScheduleType::UnorderedDynamicChunked:
110 case OMPScheduleType::UnorderedGuidedChunked:
111 case OMPScheduleType::UnorderedRuntime:
112 case OMPScheduleType::UnorderedAuto:
113 case OMPScheduleType::UnorderedTrapezoidal:
114 case OMPScheduleType::UnorderedGreedy:
115 case OMPScheduleType::UnorderedBalanced:
116 case OMPScheduleType::UnorderedGuidedIterativeChunked:
117 case OMPScheduleType::UnorderedGuidedAnalyticalChunked:
118 case OMPScheduleType::UnorderedSteal:
119 case OMPScheduleType::UnorderedStaticBalancedChunked:
120 case OMPScheduleType::UnorderedGuidedSimd:
121 case OMPScheduleType::UnorderedRuntimeSimd:
122 case OMPScheduleType::OrderedStaticChunked:
123 case OMPScheduleType::OrderedStatic:
124 case OMPScheduleType::OrderedDynamicChunked:
125 case OMPScheduleType::OrderedGuidedChunked:
126 case OMPScheduleType::OrderedRuntime:
127 case OMPScheduleType::OrderedAuto:
128 case OMPScheduleType::OrderdTrapezoidal:
129 case OMPScheduleType::NomergeUnorderedStaticChunked:
130 case OMPScheduleType::NomergeUnorderedStatic:
131 case OMPScheduleType::NomergeUnorderedDynamicChunked:
132 case OMPScheduleType::NomergeUnorderedGuidedChunked:
133 case OMPScheduleType::NomergeUnorderedRuntime:
134 case OMPScheduleType::NomergeUnorderedAuto:
135 case OMPScheduleType::NomergeUnorderedTrapezoidal:
136 case OMPScheduleType::NomergeUnorderedGreedy:
137 case OMPScheduleType::NomergeUnorderedBalanced:
138 case OMPScheduleType::NomergeUnorderedGuidedIterativeChunked:
139 case OMPScheduleType::NomergeUnorderedGuidedAnalyticalChunked:
140 case OMPScheduleType::NomergeUnorderedSteal:
141 case OMPScheduleType::NomergeOrderedStaticChunked:
142 case OMPScheduleType::NomergeOrderedStatic:
143 case OMPScheduleType::NomergeOrderedDynamicChunked:
144 case OMPScheduleType::NomergeOrderedGuidedChunked:
145 case OMPScheduleType::NomergeOrderedRuntime:
146 case OMPScheduleType::NomergeOrderedAuto:
147 case OMPScheduleType::NomergeOrderedTrapezoidal:
148 case OMPScheduleType::OrderedDistributeChunked:
149 case OMPScheduleType::OrderedDistribute:
150 break;
151 default:
152 return false;
153 }
154
155 // Must not set both monotonicity modifiers at the same time.
156 OMPScheduleType MonotonicityFlags =
157 SchedType & OMPScheduleType::MonotonicityMask;
158 if (MonotonicityFlags == OMPScheduleType::MonotonicityMask)
159 return false;
160
161 return true;
162}
163#endif
164
165/// This is a wrapper over IRBuilderBase::restoreIP that also restores a current
166/// debug location when the insert point is at the end of a block. It picks a
167/// location scoped to the current function: the block's last instruction
168/// location if the block is non-empty, otherwise a location synthesized from
169/// the function's subprogram (when the function has debug info).
170static void restoreIPandDebugLoc(llvm::IRBuilderBase &Builder,
171 llvm::IRBuilderBase::InsertPoint IP) {
172 Builder.restoreIP(IP);
173 // When IP points at a real instruction, restoreIP (SetInsertPoint) already
174 // set the debug location from that instruction, so leave it alone.
175 llvm::BasicBlock *BB = Builder.GetInsertBlock();
176 if (Builder.GetInsertPoint() != BB->end())
177 return;
178
179 // At the end of a block, pick a location guaranteed to belong to the current
180 // insertion function's subprogram. Prefer the block's own last instruction;
181 // otherwise synthesize a location from the function's subprogram.
182 if (!BB->empty())
183 Builder.SetCurrentDebugLocation(BB->back().getStableDebugLoc());
184 else if (llvm::DISubprogram *FSP =
185 BB->getParent() ? BB->getParent()->getSubprogram() : nullptr) {
186 unsigned Line = FSP->getScopeLine() ? FSP->getScopeLine() : FSP->getLine();
187 Builder.SetCurrentDebugLocation(
188 llvm::DILocation::get(Context&: FSP->getContext(), Line, /*Column=*/0, Scope: FSP));
189 }
190}
191
192static bool hasGridValue(const Triple &T) {
193 return T.isAMDGPU() || T.isNVPTX() || T.isSPIRV();
194}
195
196static const omp::GV &getGridValue(const Triple &T, Function *Kernel) {
197 if (T.isAMDGPU()) {
198 StringRef Features =
199 Kernel->getFnAttribute(Kind: "target-features").getValueAsString();
200 if (Features.count(Str: "+wavefrontsize64"))
201 return omp::getAMDGPUGridValues<64>();
202 return omp::getAMDGPUGridValues<32>();
203 }
204 if (T.isNVPTX())
205 return omp::NVPTXGridValues;
206 if (T.isSPIRV())
207 return omp::SPIRVGridValues;
208 llvm_unreachable("No grid value available for this architecture!");
209}
210
211/// Determine which scheduling algorithm to use, determined from schedule clause
212/// arguments.
213static OMPScheduleType
214getOpenMPBaseScheduleType(llvm::omp::ScheduleKind ClauseKind, bool HasChunks,
215 bool HasSimdModifier, bool HasDistScheduleChunks) {
216 // Currently, the default schedule it static.
217 switch (ClauseKind) {
218 case OMP_SCHEDULE_Default:
219 case OMP_SCHEDULE_Static:
220 return HasChunks ? OMPScheduleType::BaseStaticChunked
221 : OMPScheduleType::BaseStatic;
222 case OMP_SCHEDULE_Dynamic:
223 return OMPScheduleType::BaseDynamicChunked;
224 case OMP_SCHEDULE_Guided:
225 return HasSimdModifier ? OMPScheduleType::BaseGuidedSimd
226 : OMPScheduleType::BaseGuidedChunked;
227 case OMP_SCHEDULE_Auto:
228 return llvm::omp::OMPScheduleType::BaseAuto;
229 case OMP_SCHEDULE_Runtime:
230 return HasSimdModifier ? OMPScheduleType::BaseRuntimeSimd
231 : OMPScheduleType::BaseRuntime;
232 case OMP_SCHEDULE_Distribute:
233 return HasDistScheduleChunks ? OMPScheduleType::BaseDistributeChunked
234 : OMPScheduleType::BaseDistribute;
235 }
236 llvm_unreachable("unhandled schedule clause argument");
237}
238
239/// Adds ordering modifier flags to schedule type.
240static OMPScheduleType
241getOpenMPOrderingScheduleType(OMPScheduleType BaseScheduleType,
242 bool HasOrderedClause) {
243 assert((BaseScheduleType & OMPScheduleType::ModifierMask) ==
244 OMPScheduleType::None &&
245 "Must not have ordering nor monotonicity flags already set");
246
247 OMPScheduleType OrderingModifier = HasOrderedClause
248 ? OMPScheduleType::ModifierOrdered
249 : OMPScheduleType::ModifierUnordered;
250 OMPScheduleType OrderingScheduleType = BaseScheduleType | OrderingModifier;
251
252 // Unsupported combinations
253 if (OrderingScheduleType ==
254 (OMPScheduleType::BaseGuidedSimd | OMPScheduleType::ModifierOrdered))
255 return OMPScheduleType::OrderedGuidedChunked;
256 else if (OrderingScheduleType == (OMPScheduleType::BaseRuntimeSimd |
257 OMPScheduleType::ModifierOrdered))
258 return OMPScheduleType::OrderedRuntime;
259
260 return OrderingScheduleType;
261}
262
263/// Adds monotonicity modifier flags to schedule type.
264static OMPScheduleType
265getOpenMPMonotonicityScheduleType(OMPScheduleType ScheduleType,
266 bool HasSimdModifier, bool HasMonotonic,
267 bool HasNonmonotonic, bool HasOrderedClause) {
268 assert((ScheduleType & OMPScheduleType::MonotonicityMask) ==
269 OMPScheduleType::None &&
270 "Must not have monotonicity flags already set");
271 assert((!HasMonotonic || !HasNonmonotonic) &&
272 "Monotonic and Nonmonotonic are contradicting each other");
273
274 if (HasMonotonic) {
275 return ScheduleType | OMPScheduleType::ModifierMonotonic;
276 } else if (HasNonmonotonic) {
277 return ScheduleType | OMPScheduleType::ModifierNonmonotonic;
278 } else {
279 // OpenMP 5.1, 2.11.4 Worksharing-Loop Construct, Description.
280 // If the static schedule kind is specified or if the ordered clause is
281 // specified, and if the nonmonotonic modifier is not specified, the
282 // effect is as if the monotonic modifier is specified. Otherwise, unless
283 // the monotonic modifier is specified, the effect is as if the
284 // nonmonotonic modifier is specified.
285 OMPScheduleType BaseScheduleType =
286 ScheduleType & ~OMPScheduleType::ModifierMask;
287 if ((BaseScheduleType == OMPScheduleType::BaseStatic) ||
288 (BaseScheduleType == OMPScheduleType::BaseStaticChunked) ||
289 HasOrderedClause) {
290 // The monotonic is used by default in openmp runtime library, so no need
291 // to set it.
292 return ScheduleType;
293 } else {
294 return ScheduleType | OMPScheduleType::ModifierNonmonotonic;
295 }
296 }
297}
298
299/// Determine the schedule type using schedule and ordering clause arguments.
300static OMPScheduleType
301computeOpenMPScheduleType(ScheduleKind ClauseKind, bool HasChunks,
302 bool HasSimdModifier, bool HasMonotonicModifier,
303 bool HasNonmonotonicModifier, bool HasOrderedClause,
304 bool HasDistScheduleChunks) {
305 OMPScheduleType BaseSchedule = getOpenMPBaseScheduleType(
306 ClauseKind, HasChunks, HasSimdModifier, HasDistScheduleChunks);
307 OMPScheduleType OrderedSchedule =
308 getOpenMPOrderingScheduleType(BaseScheduleType: BaseSchedule, HasOrderedClause);
309 OMPScheduleType Result = getOpenMPMonotonicityScheduleType(
310 ScheduleType: OrderedSchedule, HasSimdModifier, HasMonotonic: HasMonotonicModifier,
311 HasNonmonotonic: HasNonmonotonicModifier, HasOrderedClause);
312
313 assert(isValidWorkshareLoopScheduleType(Result));
314 return Result;
315}
316
317/// Given a function, if it represents the entry point of a target kernel, this
318/// returns the execution mode flags associated with that kernel.
319static std::optional<omp::OMPTgtExecModeFlags>
320getTargetKernelExecMode(Function &Kernel) {
321 CallInst *TargetInitCall = nullptr;
322 for (Instruction &Inst : Kernel.getEntryBlock()) {
323 if (auto *Call = dyn_cast<CallInst>(Val: &Inst)) {
324 if (Call->getCalledFunction()->getName() == "__kmpc_target_init") {
325 TargetInitCall = Call;
326 break;
327 }
328 }
329 }
330
331 if (!TargetInitCall)
332 return std::nullopt;
333
334 // Get the kernel mode information from the global variable associated to the
335 // first argument to the call to __kmpc_target_init. Refer to
336 // createTargetInit() to see how this is initialized.
337 Value *InitOperand = TargetInitCall->getArgOperand(i: 0);
338 GlobalVariable *KernelEnv = nullptr;
339 if (auto *Cast = dyn_cast<ConstantExpr>(Val: InitOperand))
340 KernelEnv = cast<GlobalVariable>(Val: Cast->getOperand(i_nocapture: 0));
341 else
342 KernelEnv = cast<GlobalVariable>(Val: InitOperand);
343 auto *KernelEnvInit = cast<ConstantStruct>(Val: KernelEnv->getInitializer());
344 auto *ConfigEnv = cast<ConstantStruct>(Val: KernelEnvInit->getOperand(i_nocapture: 0));
345 auto *KernelMode = cast<ConstantInt>(Val: ConfigEnv->getOperand(i_nocapture: 2));
346 return static_cast<OMPTgtExecModeFlags>(KernelMode->getZExtValue());
347}
348
349static bool isGenericKernel(Function &Fn) {
350 std::optional<omp::OMPTgtExecModeFlags> ExecMode =
351 getTargetKernelExecMode(Kernel&: Fn);
352 return !ExecMode || (*ExecMode & OMP_TGT_EXEC_MODE_GENERIC);
353}
354
355/// Make \p Source branch to \p Target.
356///
357/// Handles two situations:
358/// * \p Source already has an unconditional branch.
359/// * \p Source is a degenerate block (no terminator because the BB is
360/// the current head of the IR construction).
361static void redirectTo(BasicBlock *Source, BasicBlock *Target, DebugLoc DL) {
362 if (Instruction *Term = Source->getTerminatorOrNull()) {
363 auto *Br = cast<UncondBrInst>(Val: Term);
364 BasicBlock *Succ = Br->getSuccessor();
365 Succ->removePredecessor(Pred: Source, /*KeepOneInputPHIs=*/true);
366 Br->setSuccessor(Target);
367 return;
368 }
369
370 auto *NewBr = UncondBrInst::Create(Target, InsertBefore: Source);
371 NewBr->setDebugLoc(DL);
372}
373
374void llvm::spliceBB(IRBuilderBase::InsertPoint IP, BasicBlock *New,
375 bool CreateBranch, DebugLoc DL) {
376 assert(New->getFirstInsertionPt() == New->begin() &&
377 "Target BB must not have PHI nodes");
378
379 // Move instructions to new block.
380 BasicBlock *Old = IP.getBlock();
381 // If the `Old` block is empty then there are no instructions to move. But in
382 // the new debug scheme, it could have trailing debug records which will be
383 // moved to `New` in `spliceDebugInfoEmptyBlock`. We dont want that for 2
384 // reasons:
385 // 1. If `New` is also empty, `BasicBlock::splice` crashes.
386 // 2. Even if `New` is not empty, the rationale to move those records to `New`
387 // (in `spliceDebugInfoEmptyBlock`) does not apply here. That function
388 // assumes that `Old` is optimized out and is going away. This is not the case
389 // here. The `Old` block is still being used e.g. a branch instruction is
390 // added to it later in this function.
391 // So we call `BasicBlock::splice` only when `Old` is not empty.
392 if (!Old->empty())
393 New->splice(ToIt: New->begin(), FromBB: Old, FromBeginIt: IP.getPoint(), FromEndIt: Old->end());
394
395 if (CreateBranch) {
396 auto *NewBr = UncondBrInst::Create(Target: New, InsertBefore: Old);
397 NewBr->setDebugLoc(DL);
398 }
399}
400
401void llvm::spliceBB(IRBuilder<> &Builder, BasicBlock *New, bool CreateBranch) {
402 DebugLoc DebugLoc = Builder.getCurrentDebugLocation();
403 BasicBlock *Old = Builder.GetInsertBlock();
404
405 spliceBB(IP: Builder.saveIP(), New, CreateBranch, DL: DebugLoc);
406 if (CreateBranch)
407 Builder.SetInsertPoint(Old->getTerminator());
408 else
409 Builder.SetInsertPoint(Old);
410
411 // SetInsertPoint also updates the Builder's debug location, but we want to
412 // keep the one the Builder was configured to use.
413 Builder.SetCurrentDebugLocation(DebugLoc);
414}
415
416BasicBlock *llvm::splitBB(IRBuilderBase::InsertPoint IP, bool CreateBranch,
417 DebugLoc DL, llvm::Twine Name) {
418 BasicBlock *Old = IP.getBlock();
419 BasicBlock *New = BasicBlock::Create(
420 Context&: Old->getContext(), Name: Name.isTriviallyEmpty() ? Old->getName() : Name,
421 Parent: Old->getParent(), InsertBefore: Old->getNextNode());
422 spliceBB(IP, New, CreateBranch, DL);
423 New->replaceSuccessorsPhiUsesWith(Old, New);
424 return New;
425}
426
427BasicBlock *llvm::splitBB(IRBuilderBase &Builder, bool CreateBranch,
428 llvm::Twine Name) {
429 DebugLoc DebugLoc = Builder.getCurrentDebugLocation();
430 BasicBlock *New = splitBB(IP: Builder.saveIP(), CreateBranch, DL: DebugLoc, Name);
431 if (CreateBranch)
432 Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator());
433 else
434 Builder.SetInsertPoint(Builder.GetInsertBlock());
435 // SetInsertPoint also updates the Builder's debug location, but we want to
436 // keep the one the Builder was configured to use.
437 Builder.SetCurrentDebugLocation(DebugLoc);
438 return New;
439}
440
441BasicBlock *llvm::splitBB(IRBuilder<> &Builder, bool CreateBranch,
442 llvm::Twine Name) {
443 DebugLoc DebugLoc = Builder.getCurrentDebugLocation();
444 BasicBlock *New = splitBB(IP: Builder.saveIP(), CreateBranch, DL: DebugLoc, Name);
445 if (CreateBranch)
446 Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator());
447 else
448 Builder.SetInsertPoint(Builder.GetInsertBlock());
449 // SetInsertPoint also updates the Builder's debug location, but we want to
450 // keep the one the Builder was configured to use.
451 Builder.SetCurrentDebugLocation(DebugLoc);
452 return New;
453}
454
455BasicBlock *llvm::splitBBWithSuffix(IRBuilderBase &Builder, bool CreateBranch,
456 llvm::Twine Suffix) {
457 BasicBlock *Old = Builder.GetInsertBlock();
458 return splitBB(Builder, CreateBranch, Name: Old->getName() + Suffix);
459}
460
461// This function creates a fake integer value and a fake use for the integer
462// value. It returns the fake value created. This is useful in modeling the
463// extra arguments to the outlined functions.
464Value *createFakeIntVal(IRBuilderBase &Builder,
465 OpenMPIRBuilder::InsertPointTy OuterAllocaIP,
466 llvm::SmallVectorImpl<Instruction *> &ToBeDeleted,
467 OpenMPIRBuilder::InsertPointTy InnerAllocaIP,
468 const Twine &Name = "", bool AsPtr = true,
469 bool Is64Bit = false) {
470 Builder.restoreIP(IP: OuterAllocaIP);
471 IntegerType *IntTy = Is64Bit ? Builder.getInt64Ty() : Builder.getInt32Ty();
472 Instruction *FakeVal;
473 AllocaInst *FakeValAddr =
474 Builder.CreateAlloca(Ty: IntTy, ArraySize: nullptr, Name: Name + ".addr");
475 ToBeDeleted.push_back(Elt: FakeValAddr);
476
477 if (AsPtr) {
478 FakeVal = FakeValAddr;
479 } else {
480 FakeVal = Builder.CreateLoad(Ty: IntTy, Ptr: FakeValAddr, Name: Name + ".val");
481 ToBeDeleted.push_back(Elt: FakeVal);
482 }
483
484 // Generate a fake use of this value
485 Builder.restoreIP(IP: InnerAllocaIP);
486 Instruction *UseFakeVal;
487 if (AsPtr) {
488 UseFakeVal = Builder.CreateLoad(Ty: IntTy, Ptr: FakeVal, Name: Name + ".use");
489 } else {
490 UseFakeVal = cast<BinaryOperator>(Val: Builder.CreateAdd(
491 LHS: FakeVal, RHS: Is64Bit ? Builder.getInt64(C: 10) : Builder.getInt32(C: 10)));
492 }
493 ToBeDeleted.push_back(Elt: UseFakeVal);
494 return FakeVal;
495}
496
497//===----------------------------------------------------------------------===//
498// OpenMPIRBuilderConfig
499//===----------------------------------------------------------------------===//
500
501namespace {
502LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();
503/// Values for bit flags for marking which requires clauses have been used.
504enum OpenMPOffloadingRequiresDirFlags {
505 /// flag undefined.
506 OMP_REQ_UNDEFINED = 0x000,
507 /// no requires directive present.
508 OMP_REQ_NONE = 0x001,
509 /// reverse_offload clause.
510 OMP_REQ_REVERSE_OFFLOAD = 0x002,
511 /// unified_address clause.
512 OMP_REQ_UNIFIED_ADDRESS = 0x004,
513 /// unified_shared_memory clause.
514 OMP_REQ_UNIFIED_SHARED_MEMORY = 0x008,
515 /// dynamic_allocators clause.
516 OMP_REQ_DYNAMIC_ALLOCATORS = 0x010,
517 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_REQ_DYNAMIC_ALLOCATORS)
518};
519
520class OMPCodeExtractor : public CodeExtractor {
521public:
522 OMPCodeExtractor(OpenMPIRBuilder &OMPBuilder, ArrayRef<BasicBlock *> BBs,
523 DominatorTree *DT = nullptr, bool AggregateArgs = false,
524 BlockFrequencyInfo *BFI = nullptr,
525 BranchProbabilityInfo *BPI = nullptr,
526 AssumptionCache *AC = nullptr, bool AllowVarArgs = false,
527 bool AllowAlloca = false,
528 BasicBlock *AllocationBlock = nullptr,
529 ArrayRef<BasicBlock *> DeallocationBlocks = {},
530 std::string Suffix = "", bool ArgsInZeroAddressSpace = false)
531 : CodeExtractor(BBs, DT, AggregateArgs, BFI, BPI, AC, AllowVarArgs,
532 AllowAlloca, AllocationBlock, DeallocationBlocks, Suffix,
533 ArgsInZeroAddressSpace),
534 OMPBuilder(OMPBuilder) {}
535
536 virtual ~OMPCodeExtractor() = default;
537
538protected:
539 OpenMPIRBuilder &OMPBuilder;
540};
541
542class DeviceSharedMemCodeExtractor : public OMPCodeExtractor {
543public:
544 using OMPCodeExtractor::OMPCodeExtractor;
545 virtual ~DeviceSharedMemCodeExtractor() = default;
546
547protected:
548 virtual Instruction *
549 allocateVar(IRBuilder<>::InsertPoint AllocaIP, Type *VarType,
550 const Twine &Name = Twine(""),
551 AddrSpaceCastInst **CastedAlloc = nullptr) override {
552 return OMPBuilder.createOMPAllocShared(Loc: AllocaIP, VarType, Name);
553 }
554
555 virtual Instruction *deallocateVar(IRBuilder<>::InsertPoint DeallocIP,
556 Value *Var, Type *VarType) override {
557 return OMPBuilder.createOMPFreeShared(Loc: DeallocIP, Addr: Var, VarType);
558 }
559};
560
561/// Helper storing information about regions to outline using device shared
562/// memory for intermediate allocations.
563struct DeviceSharedMemOutlineInfo : public OpenMPIRBuilder::OutlineInfo {
564 OpenMPIRBuilder &OMPBuilder;
565
566 DeviceSharedMemOutlineInfo(OpenMPIRBuilder &OMPBuilder)
567 : OMPBuilder(OMPBuilder) {}
568 virtual ~DeviceSharedMemOutlineInfo() = default;
569
570 virtual std::unique_ptr<CodeExtractor>
571 createCodeExtractor(ArrayRef<BasicBlock *> Blocks,
572 bool ArgsInZeroAddressSpace,
573 Twine Suffix = Twine("")) override;
574};
575
576} // anonymous namespace
577
578OpenMPIRBuilderConfig::OpenMPIRBuilderConfig()
579 : RequiresFlags(OMP_REQ_UNDEFINED) {}
580
581OpenMPIRBuilderConfig::OpenMPIRBuilderConfig(
582 bool IsTargetDevice, bool IsGPU, bool OpenMPOffloadMandatory,
583 bool HasRequiresReverseOffload, bool HasRequiresUnifiedAddress,
584 bool HasRequiresUnifiedSharedMemory, bool HasRequiresDynamicAllocators)
585 : IsTargetDevice(IsTargetDevice), IsGPU(IsGPU),
586 OpenMPOffloadMandatory(OpenMPOffloadMandatory),
587 RequiresFlags(OMP_REQ_UNDEFINED) {
588 if (HasRequiresReverseOffload)
589 RequiresFlags |= OMP_REQ_REVERSE_OFFLOAD;
590 if (HasRequiresUnifiedAddress)
591 RequiresFlags |= OMP_REQ_UNIFIED_ADDRESS;
592 if (HasRequiresUnifiedSharedMemory)
593 RequiresFlags |= OMP_REQ_UNIFIED_SHARED_MEMORY;
594 if (HasRequiresDynamicAllocators)
595 RequiresFlags |= OMP_REQ_DYNAMIC_ALLOCATORS;
596}
597
598bool OpenMPIRBuilderConfig::hasRequiresReverseOffload() const {
599 return RequiresFlags & OMP_REQ_REVERSE_OFFLOAD;
600}
601
602bool OpenMPIRBuilderConfig::hasRequiresUnifiedAddress() const {
603 return RequiresFlags & OMP_REQ_UNIFIED_ADDRESS;
604}
605
606bool OpenMPIRBuilderConfig::hasRequiresUnifiedSharedMemory() const {
607 return RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY;
608}
609
610bool OpenMPIRBuilderConfig::hasRequiresDynamicAllocators() const {
611 return RequiresFlags & OMP_REQ_DYNAMIC_ALLOCATORS;
612}
613
614int64_t OpenMPIRBuilderConfig::getRequiresFlags() const {
615 return hasRequiresFlags() ? RequiresFlags
616 : static_cast<int64_t>(OMP_REQ_NONE);
617}
618
619void OpenMPIRBuilderConfig::setHasRequiresReverseOffload(bool Value) {
620 if (Value)
621 RequiresFlags |= OMP_REQ_REVERSE_OFFLOAD;
622 else
623 RequiresFlags &= ~OMP_REQ_REVERSE_OFFLOAD;
624}
625
626void OpenMPIRBuilderConfig::setHasRequiresUnifiedAddress(bool Value) {
627 if (Value)
628 RequiresFlags |= OMP_REQ_UNIFIED_ADDRESS;
629 else
630 RequiresFlags &= ~OMP_REQ_UNIFIED_ADDRESS;
631}
632
633void OpenMPIRBuilderConfig::setHasRequiresUnifiedSharedMemory(bool Value) {
634 if (Value)
635 RequiresFlags |= OMP_REQ_UNIFIED_SHARED_MEMORY;
636 else
637 RequiresFlags &= ~OMP_REQ_UNIFIED_SHARED_MEMORY;
638}
639
640void OpenMPIRBuilderConfig::setHasRequiresDynamicAllocators(bool Value) {
641 if (Value)
642 RequiresFlags |= OMP_REQ_DYNAMIC_ALLOCATORS;
643 else
644 RequiresFlags &= ~OMP_REQ_DYNAMIC_ALLOCATORS;
645}
646
647//===----------------------------------------------------------------------===//
648// OpenMPIRBuilder
649//===----------------------------------------------------------------------===//
650
651void OpenMPIRBuilder::getKernelArgsVector(TargetKernelArgs &KernelArgs,
652 IRBuilderBase &Builder,
653 SmallVector<Value *> &ArgsVector) {
654 Value *Version = Builder.getInt32(OMP_KERNEL_ARG_VERSION);
655 Value *PointerNum = Builder.getInt32(C: KernelArgs.NumTargetItems);
656 auto Int32Ty = Type::getInt32Ty(C&: Builder.getContext());
657 constexpr size_t MaxDim = 3;
658 Value *ZeroArray = Constant::getNullValue(Ty: ArrayType::get(ElementType: Int32Ty, NumElements: MaxDim));
659
660 Value *HasNoWaitFlag = Builder.getInt64(C: KernelArgs.HasNoWait);
661
662 Value *DynCGroupMemFallbackFlag =
663 Builder.getInt64(C: static_cast<uint64_t>(KernelArgs.DynCGroupMemFallback));
664 DynCGroupMemFallbackFlag = Builder.CreateShl(LHS: DynCGroupMemFallbackFlag, RHS: 2);
665
666 Value *StrictBlocksFlag = Builder.getInt64(C: KernelArgs.StrictBlocks);
667 Value *StrictThreadsFlag = Builder.getInt64(C: KernelArgs.StrictThreads);
668
669 StrictBlocksFlag = Builder.CreateShl(LHS: StrictBlocksFlag, RHS: 6);
670 StrictThreadsFlag = Builder.CreateShl(LHS: StrictThreadsFlag, RHS: 7);
671
672 Value *Flags = Builder.CreateOr(LHS: HasNoWaitFlag, RHS: DynCGroupMemFallbackFlag);
673 Flags = Builder.CreateOr(LHS: Flags, RHS: StrictBlocksFlag);
674 Flags = Builder.CreateOr(LHS: Flags, RHS: StrictThreadsFlag);
675
676 assert(!KernelArgs.NumTeams.empty() && !KernelArgs.NumThreads.empty());
677
678 Value *NumTeams3D =
679 Builder.CreateInsertValue(Agg: ZeroArray, Val: KernelArgs.NumTeams[0], Idxs: {0});
680 Value *NumThreads3D =
681 Builder.CreateInsertValue(Agg: ZeroArray, Val: KernelArgs.NumThreads[0], Idxs: {0});
682 for (unsigned I :
683 seq<unsigned>(Begin: 1, End: std::min(a: KernelArgs.NumTeams.size(), b: MaxDim)))
684 NumTeams3D =
685 Builder.CreateInsertValue(Agg: NumTeams3D, Val: KernelArgs.NumTeams[I], Idxs: {I});
686 for (unsigned I :
687 seq<unsigned>(Begin: 1, End: std::min(a: KernelArgs.NumThreads.size(), b: MaxDim)))
688 NumThreads3D =
689 Builder.CreateInsertValue(Agg: NumThreads3D, Val: KernelArgs.NumThreads[I], Idxs: {I});
690
691 ArgsVector = {Version,
692 PointerNum,
693 KernelArgs.RTArgs.BasePointersArray,
694 KernelArgs.RTArgs.PointersArray,
695 KernelArgs.RTArgs.SizesArray,
696 KernelArgs.RTArgs.MapTypesArray,
697 KernelArgs.RTArgs.MapNamesArray,
698 KernelArgs.RTArgs.MappersArray,
699 KernelArgs.NumIterations,
700 Flags,
701 NumTeams3D,
702 NumThreads3D,
703 KernelArgs.DynCGroupMem};
704}
705
706void OpenMPIRBuilder::addAttributes(omp::RuntimeFunction FnID, Function &Fn) {
707 LLVMContext &Ctx = Fn.getContext();
708
709 // Get the function's current attributes.
710 auto Attrs = Fn.getAttributes();
711 auto FnAttrs = Attrs.getFnAttrs();
712 auto RetAttrs = Attrs.getRetAttrs();
713 SmallVector<AttributeSet, 4> ArgAttrs;
714 for (size_t ArgNo = 0; ArgNo < Fn.arg_size(); ++ArgNo)
715 ArgAttrs.emplace_back(Args: Attrs.getParamAttrs(ArgNo));
716
717 // Add AS to FnAS while taking special care with integer extensions.
718 auto addAttrSet = [&](AttributeSet &FnAS, const AttributeSet &AS,
719 bool Param = true) -> void {
720 bool HasSignExt = AS.hasAttribute(Kind: Attribute::SExt);
721 bool HasZeroExt = AS.hasAttribute(Kind: Attribute::ZExt);
722 if (HasSignExt || HasZeroExt) {
723 assert(AS.getNumAttributes() == 1 &&
724 "Currently not handling extension attr combined with others.");
725 if (Param) {
726 if (auto AK = TargetLibraryInfo::getExtAttrForI32Param(T, Signed: HasSignExt))
727 FnAS = FnAS.addAttribute(C&: Ctx, Kind: AK);
728 } else if (auto AK =
729 TargetLibraryInfo::getExtAttrForI32Return(T, Signed: HasSignExt))
730 FnAS = FnAS.addAttribute(C&: Ctx, Kind: AK);
731 } else {
732 FnAS = FnAS.addAttributes(C&: Ctx, AS);
733 }
734 };
735
736#define OMP_ATTRS_SET(VarName, AttrSet) AttributeSet VarName = AttrSet;
737#include "llvm/Frontend/OpenMP/OMPKinds.def"
738
739 // Add attributes to the function declaration.
740 switch (FnID) {
741#define OMP_RTL_ATTRS(Enum, FnAttrSet, RetAttrSet, ArgAttrSets) \
742 case Enum: \
743 FnAttrs = FnAttrs.addAttributes(Ctx, FnAttrSet); \
744 addAttrSet(RetAttrs, RetAttrSet, /*Param*/ false); \
745 for (size_t ArgNo = 0; ArgNo < ArgAttrSets.size(); ++ArgNo) \
746 addAttrSet(ArgAttrs[ArgNo], ArgAttrSets[ArgNo]); \
747 Fn.setAttributes(AttributeList::get(Ctx, FnAttrs, RetAttrs, ArgAttrs)); \
748 break;
749#include "llvm/Frontend/OpenMP/OMPKinds.def"
750 default:
751 // Attributes are optional.
752 break;
753 }
754}
755
756FunctionCallee
757OpenMPIRBuilder::getOrCreateRuntimeFunction(Module &M, RuntimeFunction FnID) {
758 FunctionType *FnTy = nullptr;
759 Function *Fn = nullptr;
760
761 // Try to find the declation in the module first.
762 switch (FnID) {
763#define OMP_RTL(Enum, Str, IsVarArg, ReturnType, ...) \
764 case Enum: \
765 FnTy = FunctionType::get(ReturnType, ArrayRef<Type *>{__VA_ARGS__}, \
766 IsVarArg); \
767 Fn = M.getFunction(Str); \
768 break;
769#include "llvm/Frontend/OpenMP/OMPKinds.def"
770 }
771
772 if (!Fn) {
773 // Create a new declaration if we need one.
774 switch (FnID) {
775#define OMP_RTL(Enum, Str, ...) \
776 case Enum: \
777 Fn = Function::Create(FnTy, GlobalValue::ExternalLinkage, Str, M); \
778 break;
779#include "llvm/Frontend/OpenMP/OMPKinds.def"
780 }
781 Fn->setCallingConv(Config.getRuntimeCC());
782 // Add information if the runtime function takes a callback function
783 if (FnID == OMPRTL___kmpc_fork_call || FnID == OMPRTL___kmpc_fork_teams) {
784 if (!Fn->hasMetadata(KindID: LLVMContext::MD_callback)) {
785 LLVMContext &Ctx = Fn->getContext();
786 MDBuilder MDB(Ctx);
787 // Annotate the callback behavior of the runtime function:
788 // - The callback callee is argument number 2 (microtask).
789 // - The first two arguments of the callback callee are unknown (-1).
790 // - All variadic arguments to the runtime function are passed to the
791 // callback callee.
792 Fn->addMetadata(
793 KindID: LLVMContext::MD_callback,
794 MD&: *MDNode::get(Context&: Ctx, MDs: {MDB.createCallbackEncoding(
795 CalleeArgNo: 2, Arguments: {-1, -1}, /* VarArgsArePassed */ true)}));
796 }
797 }
798
799 LLVM_DEBUG(dbgs() << "Created OpenMP runtime function " << Fn->getName()
800 << " with type " << *Fn->getFunctionType() << "\n");
801 addAttributes(FnID, Fn&: *Fn);
802
803 } else {
804 LLVM_DEBUG(dbgs() << "Found OpenMP runtime function " << Fn->getName()
805 << " with type " << *Fn->getFunctionType() << "\n");
806 }
807
808 assert(Fn && "Failed to create OpenMP runtime function");
809
810 return {FnTy, Fn};
811}
812
813Expected<BasicBlock *>
814OpenMPIRBuilder::FinalizationInfo::getFiniBB(IRBuilderBase &Builder) {
815 if (!FiniBB) {
816 Function *ParentFunc = Builder.GetInsertBlock()->getParent();
817 IRBuilderBase::InsertPointGuard Guard(Builder);
818 FiniBB = BasicBlock::Create(Context&: Builder.getContext(), Name: ".fini", Parent: ParentFunc);
819 Builder.SetInsertPoint(FiniBB);
820 // FiniCB adds the branch to the exit stub.
821 if (Error Err = FiniCB(Builder.saveIP()))
822 return Err;
823 }
824 return FiniBB;
825}
826
827Error OpenMPIRBuilder::FinalizationInfo::mergeFiniBB(IRBuilderBase &Builder,
828 BasicBlock *OtherFiniBB) {
829 // Simple case: FiniBB does not exist yet: re-use OtherFiniBB.
830 if (!FiniBB) {
831 FiniBB = OtherFiniBB;
832
833 Builder.SetInsertPoint(FiniBB->getFirstNonPHIIt());
834 if (Error Err = FiniCB(Builder.saveIP()))
835 return Err;
836
837 return Error::success();
838 }
839
840 // Move instructions from FiniBB to the start of OtherFiniBB.
841 auto EndIt = FiniBB->end();
842 if (FiniBB->size() >= 1)
843 if (auto Prev = std::prev(x: EndIt); Prev->isTerminator())
844 EndIt = Prev;
845 OtherFiniBB->splice(ToIt: OtherFiniBB->getFirstNonPHIIt(), FromBB: FiniBB, FromBeginIt: FiniBB->begin(),
846 FromEndIt: EndIt);
847
848 FiniBB->replaceAllUsesWith(V: OtherFiniBB);
849 FiniBB->eraseFromParent();
850 FiniBB = OtherFiniBB;
851 return Error::success();
852}
853
854Function *OpenMPIRBuilder::getOrCreateRuntimeFunctionPtr(RuntimeFunction FnID) {
855 FunctionCallee RTLFn = getOrCreateRuntimeFunction(M, FnID);
856 auto *Fn = dyn_cast<llvm::Function>(Val: RTLFn.getCallee());
857 assert(Fn && "Failed to create OpenMP runtime function pointer");
858 return Fn;
859}
860
861CallInst *OpenMPIRBuilder::createRuntimeFunctionCall(FunctionCallee Callee,
862 ArrayRef<Value *> Args,
863 StringRef Name) {
864 CallInst *Call = Builder.CreateCall(Callee, Args, Name);
865 Call->setCallingConv(Config.getRuntimeCC());
866 return Call;
867}
868
869void OpenMPIRBuilder::initialize() { initializeTypes(M); }
870
871static void raiseUserConstantDataAllocasToEntryBlock(IRBuilderBase &Builder,
872 Function *Function) {
873 BasicBlock &EntryBlock = Function->getEntryBlock();
874 BasicBlock::iterator MoveLocInst = EntryBlock.getFirstNonPHIIt();
875
876 // Loop over blocks looking for constant allocas, skipping the entry block
877 // as any allocas there are already in the desired location.
878 for (auto Block = std::next(x: Function->begin(), n: 1); Block != Function->end();
879 Block++) {
880 for (auto Inst = Block->getReverseIterator()->begin();
881 Inst != Block->getReverseIterator()->end();) {
882 if (auto *AllocaInst = dyn_cast_if_present<llvm::AllocaInst>(Val&: Inst)) {
883 Inst++;
884 if (!isa<ConstantData>(Val: AllocaInst->getArraySize()))
885 continue;
886 AllocaInst->moveBeforePreserving(MovePos: MoveLocInst);
887 } else {
888 Inst++;
889 }
890 }
891 }
892}
893
894static void hoistNonEntryAllocasToEntryBlock(llvm::BasicBlock &Block) {
895 llvm::SmallVector<llvm::Instruction *> AllocasToMove;
896
897 auto ShouldHoistAlloca = [](const llvm::AllocaInst &AllocaInst) {
898 // TODO: For now, we support simple static allocations, we might need to
899 // move non-static ones as well. However, this will need further analysis to
900 // move the lenght arguments as well.
901 return !AllocaInst.isArrayAllocation();
902 };
903
904 for (llvm::Instruction &Inst : Block)
905 if (auto *AllocaInst = llvm::dyn_cast<llvm::AllocaInst>(Val: &Inst))
906 if (ShouldHoistAlloca(*AllocaInst))
907 AllocasToMove.push_back(Elt: AllocaInst);
908
909 auto InsertPoint =
910 Block.getParent()->getEntryBlock().getTerminator()->getIterator();
911
912 for (llvm::Instruction *AllocaInst : AllocasToMove)
913 AllocaInst->moveBefore(InsertPos: InsertPoint);
914}
915
916static void hoistNonEntryAllocasToEntryBlock(llvm::Function *Func) {
917 PostDominatorTree PostDomTree(*Func);
918 for (llvm::BasicBlock &BB : *Func)
919 if (PostDomTree.properlyDominates(A: &BB, B: &Func->getEntryBlock()))
920 hoistNonEntryAllocasToEntryBlock(Block&: BB);
921}
922
923void OpenMPIRBuilder::finalize(Function *Fn) {
924 SmallPtrSet<BasicBlock *, 32> ParallelRegionBlockSet;
925 SmallVector<BasicBlock *, 32> Blocks;
926 SmallVector<std::unique_ptr<OutlineInfo>, 16> DeferredOutlines;
927 for (std::unique_ptr<OutlineInfo> &OI : OutlineInfos) {
928 // Skip functions that have not finalized yet; may happen with nested
929 // function generation.
930 if (Fn && OI->getFunction() != Fn) {
931 DeferredOutlines.push_back(Elt: std::move(OI));
932 continue;
933 }
934
935 ParallelRegionBlockSet.clear();
936 Blocks.clear();
937 OI->collectBlocks(BlockSet&: ParallelRegionBlockSet, BlockVector&: Blocks);
938
939 Function *OuterFn = OI->getFunction();
940 CodeExtractorAnalysisCache CEAC(*OuterFn);
941 // If we generate code for the target device, we need to allocate
942 // struct for aggregate params in the device default alloca address space.
943 // OpenMP runtime requires that the params of the extracted functions are
944 // passed as zero address space pointers. This flag ensures that
945 // CodeExtractor generates correct code for extracted functions
946 // which are used by OpenMP runtime.
947 bool ArgsInZeroAddressSpace = Config.isTargetDevice();
948 std::unique_ptr<CodeExtractor> Extractor =
949 OI->createCodeExtractor(Blocks, ArgsInZeroAddressSpace, Suffix: ".omp_par");
950
951 LLVM_DEBUG(dbgs() << "Before outlining: " << *OuterFn << "\n");
952 LLVM_DEBUG(dbgs() << "Entry " << OI->EntryBB->getName()
953 << " Exit: " << OI->ExitBB->getName() << "\n");
954 assert(Extractor->isEligible() &&
955 "Expected OpenMP outlining to be possible!");
956
957 for (auto *V : OI->ExcludeArgsFromAggregate)
958 Extractor->excludeArgFromAggregate(Arg: V);
959
960 Function *OutlinedFn =
961 Extractor->extractCodeRegion(CEAC, Inputs&: OI->Inputs, Outputs&: OI->Outputs);
962
963 // Forward target-cpu, target-features attributes to the outlined function.
964 auto TargetCpuAttr = OuterFn->getFnAttribute(Kind: "target-cpu");
965 if (TargetCpuAttr.isStringAttribute())
966 OutlinedFn->addFnAttr(Attr: TargetCpuAttr);
967
968 auto TargetFeaturesAttr = OuterFn->getFnAttribute(Kind: "target-features");
969 if (TargetFeaturesAttr.isStringAttribute())
970 OutlinedFn->addFnAttr(Attr: TargetFeaturesAttr);
971
972 LLVM_DEBUG(dbgs() << "After outlining: " << *OuterFn << "\n");
973 LLVM_DEBUG(dbgs() << " Outlined function: " << *OutlinedFn << "\n");
974 assert(OutlinedFn->getReturnType()->isVoidTy() &&
975 "OpenMP outlined functions should not return a value!");
976
977 // For compability with the clang CG we move the outlined function after the
978 // one with the parallel region.
979 OutlinedFn->removeFromParent();
980 M.getFunctionList().insertAfter(where: OuterFn->getIterator(), New: OutlinedFn);
981
982 // Remove the artificial entry introduced by the extractor right away, we
983 // made our own entry block after all.
984 {
985 BasicBlock &ArtificialEntry = OutlinedFn->getEntryBlock();
986 assert(ArtificialEntry.getUniqueSuccessor() == OI->EntryBB);
987 assert(OI->EntryBB->getUniquePredecessor() == &ArtificialEntry);
988 // Move instructions from the to-be-deleted ArtificialEntry to the entry
989 // basic block of the parallel region. CodeExtractor generates
990 // instructions to unwrap the aggregate argument and may sink
991 // allocas/bitcasts for values that are solely used in the outlined region
992 // and do not escape.
993 assert(!ArtificialEntry.empty() &&
994 "Expected instructions to add in the outlined region entry");
995 for (BasicBlock::reverse_iterator It = ArtificialEntry.rbegin(),
996 End = ArtificialEntry.rend();
997 It != End;) {
998 Instruction &I = *It;
999 It++;
1000
1001 if (I.isTerminator()) {
1002 // Absorb any debug value that terminator may have
1003 if (Instruction *TI = OI->EntryBB->getTerminatorOrNull())
1004 TI->adoptDbgRecords(BB: &ArtificialEntry, It: I.getIterator(), InsertAtHead: false);
1005 continue;
1006 }
1007
1008 I.moveBeforePreserving(BB&: *OI->EntryBB,
1009 I: OI->EntryBB->getFirstInsertionPt());
1010 }
1011
1012 OI->EntryBB->moveBefore(MovePos: &ArtificialEntry);
1013 ArtificialEntry.eraseFromParent();
1014 }
1015 assert(&OutlinedFn->getEntryBlock() == OI->EntryBB);
1016 assert(OutlinedFn && OutlinedFn->hasNUses(1));
1017
1018 // Run a user callback, e.g. to add attributes.
1019 if (OI->PostOutlineCB)
1020 OI->PostOutlineCB(*OutlinedFn);
1021
1022 if (OI->FixUpNonEntryAllocas)
1023 hoistNonEntryAllocasToEntryBlock(Func: OutlinedFn);
1024 }
1025
1026 // Remove work items that have been completed.
1027 OutlineInfos = std::move(DeferredOutlines);
1028
1029 // The createTarget functions embeds user written code into
1030 // the target region which may inject allocas which need to
1031 // be moved to the entry block of our target or risk malformed
1032 // optimisations by later passes, this is only relevant for
1033 // the device pass which appears to be a little more delicate
1034 // when it comes to optimisations (however, we do not block on
1035 // that here, it's up to the inserter to the list to do so).
1036 // This notbaly has to occur after the OutlinedInfo candidates
1037 // have been extracted so we have an end product that will not
1038 // be implicitly adversely affected by any raises unless
1039 // intentionally appended to the list.
1040 // NOTE: This only does so for ConstantData, it could be extended
1041 // to ConstantExpr's with further effort, however, they should
1042 // largely be folded when they get here. Extending it to runtime
1043 // defined/read+writeable allocation sizes would be non-trivial
1044 // (need to factor in movement of any stores to variables the
1045 // allocation size depends on, as well as the usual loads,
1046 // otherwise it'll yield the wrong result after movement) and
1047 // likely be more suitable as an LLVM optimisation pass.
1048 for (Function *F : ConstantAllocaRaiseCandidates)
1049 raiseUserConstantDataAllocasToEntryBlock(Builder, Function: F);
1050
1051 EmitMetadataErrorReportFunctionTy &&ErrorReportFn =
1052 [](EmitMetadataErrorKind Kind,
1053 const TargetRegionEntryInfo &EntryInfo) -> void {
1054 errs() << "Error of kind: " << Kind
1055 << " when emitting offload entries and metadata during "
1056 "OMPIRBuilder finalization \n";
1057 };
1058
1059 if (!OffloadInfoManager.empty())
1060 createOffloadEntriesAndInfoMetadata(ErrorReportFunction&: ErrorReportFn);
1061
1062 // Rewrite uses of globals to their replacement declare target globals if
1063 // we are processing a device module.
1064 if (Config.isTargetDevice())
1065 applyDeclareTargetGlobalReplacements();
1066
1067 if (Config.EmitLLVMUsedMetaInfo.value_or(u: false)) {
1068 std::vector<WeakTrackingVH> LLVMCompilerUsed = {
1069 M.getGlobalVariable(Name: "__openmp_nvptx_data_transfer_temporary_storage")};
1070 emitUsed(Name: "llvm.compiler.used", List: LLVMCompilerUsed);
1071 }
1072
1073 IsFinalized = true;
1074}
1075
1076bool OpenMPIRBuilder::isFinalized() { return IsFinalized; }
1077
1078void OpenMPIRBuilder::registerDeclareTargetGlobalReplacement(
1079 GlobalValue *Original, GlobalValue *Replacement) {
1080 assert(Original && Replacement &&
1081 "Null values provided to registerDeclareTargetGlobalReplacement");
1082 DeclareTargetGlobalReplacements.push_back(Elt: {.Original: Original, .Replacement: Replacement});
1083}
1084
1085void OpenMPIRBuilder::applyDeclareTargetGlobalReplacements() {
1086 for (DeclareTargetGlobalReplacement &R : DeclareTargetGlobalReplacements) {
1087 GlobalValue *OldGV = R.Original;
1088 GlobalValue *NewGV = R.Replacement;
1089
1090 assert(OldGV && NewGV &&
1091 "A null value was inserted into DeclareTargetGlobalReplacements");
1092
1093 // The assert above should catch this case, but this is kept to attempt
1094 // to proceed without issue when asserts are off.
1095 if (!OldGV || !NewGV)
1096 continue;
1097
1098 // The replacement global is a reference pointer that holds the
1099 // address of the device-resident storage. Every use must load the
1100 // reference pointer first and use the loaded address.
1101 //
1102 // Constant expression users (e.g. a constant GEP embedded in another
1103 // global's initializer or in an instruction) cannot have a load inserted
1104 // in place, so first expand any constant-expression users that live inside
1105 // functions into instructions. Any remaining constant users are handled
1106 // via a direct constant rewrite below as we cannot materialize a load
1107 // there.
1108 //
1109 // NOTE: We extend the constant rewrite to module scope, as we replace all
1110 // usages.
1111 if (auto *OldConst = dyn_cast<Constant>(Val: OldGV))
1112 convertUsersOfConstantsToInstructions(Consts: OldConst,
1113 /*RestrictToFunc=*/nullptr,
1114 /*RemoveDeadConstants=*/false);
1115
1116 IRBuilderBase::InsertPointGuard Guard(Builder);
1117 SmallVector<User *, 16> Users(OldGV->users());
1118 for (User *U : Users) {
1119 auto *Insn = dyn_cast<Instruction>(Val: U);
1120 if (!Insn)
1121 continue;
1122
1123 // A PHI node cannot have a load inserted immediately before it, as PHIs
1124 // must remain grouped at the top of their basic block. So we need to
1125 // make sure any loads we emit are generated in the preceding edge, a
1126 // PHI may reference the global on more than one edge, so every matching
1127 // slot must be handled.
1128 if (auto *PHI = dyn_cast<PHINode>(Val: Insn)) {
1129 for (unsigned I = 0, E = PHI->getNumIncomingValues(); I < E; ++I) {
1130 if (PHI->getIncomingValue(i: I) != OldGV)
1131 continue;
1132
1133 BasicBlock *IncomingBB = PHI->getIncomingBlock(i: I);
1134 Builder.SetInsertPoint(IncomingBB->getTerminator());
1135 Builder.SetCurrentDebugLocation(PHI->getDebugLoc());
1136 LoadInst *EdgeLoad = Builder.CreateLoad(Ty: NewGV->getType(), Ptr: NewGV);
1137 PHI->setIncomingValue(i: I, V: EdgeLoad);
1138 }
1139 continue;
1140 }
1141
1142 Builder.SetInsertPoint(Insn);
1143 Builder.SetCurrentDebugLocation(Insn->getDebugLoc());
1144 LoadInst *Load = Builder.CreateLoad(Ty: NewGV->getType(), Ptr: NewGV);
1145
1146 // The replacement declare target global lives in the default address
1147 // space, whereas the original global may reside in a non-default
1148 // address space. In that case the initial lowering may have
1149 // emitted an addrspacecast that is no longer valid. Replace the
1150 // whole addrspacecast with the load and erase it rather than
1151 // feeding the load back into the (now pointless) cast.
1152 // NOTE: If we end up with replacement declare target globals in
1153 // non-zero AS's the below will need some minor extensions to have the
1154 // option to alter the address space cast to the new address space where
1155 // required rather than just replacing it.
1156 if (auto *ASC = dyn_cast<AddrSpaceCastInst>(Val: Insn)) {
1157 unsigned NewGVAS = NewGV->getType()->getPointerAddressSpace();
1158 assert(NewGVAS == 0 &&
1159 "Non-default address space declare target global");
1160 unsigned OldGVAS = OldGV->getType()->getPointerAddressSpace();
1161 unsigned DestAS = ASC->getType()->getPointerAddressSpace();
1162 if (DestAS == 0 && NewGVAS != OldGVAS) {
1163 ASC->replaceAllUsesWith(V: Load);
1164 ASC->eraseFromParent();
1165 continue;
1166 }
1167 }
1168
1169 Insn->replaceUsesOfWith(From: OldGV, To: Load);
1170 }
1171 }
1172
1173 DeclareTargetGlobalReplacements.clear();
1174}
1175
1176OpenMPIRBuilder::~OpenMPIRBuilder() {
1177 assert(OutlineInfos.empty() && "There must be no outstanding outlinings");
1178}
1179
1180GlobalValue *OpenMPIRBuilder::createGlobalFlag(unsigned Value, StringRef Name) {
1181 IntegerType *I32Ty = Type::getInt32Ty(C&: M.getContext());
1182 auto *GV =
1183 new GlobalVariable(M, I32Ty,
1184 /* isConstant = */ true, GlobalValue::WeakODRLinkage,
1185 ConstantInt::get(Ty: I32Ty, V: Value), Name);
1186 GV->setVisibility(GlobalValue::HiddenVisibility);
1187
1188 return GV;
1189}
1190
1191void OpenMPIRBuilder::emitUsed(StringRef Name, ArrayRef<WeakTrackingVH> List) {
1192 if (List.empty())
1193 return;
1194
1195 // Convert List to what ConstantArray needs.
1196 SmallVector<Constant *, 8> UsedArray;
1197 UsedArray.resize(N: List.size());
1198 for (unsigned I = 0, E = List.size(); I != E; ++I)
1199 UsedArray[I] = ConstantExpr::getPointerBitCastOrAddrSpaceCast(
1200 C: cast<Constant>(Val: &*List[I]), Ty: Builder.getPtrTy());
1201
1202 if (UsedArray.empty())
1203 return;
1204 ArrayType *ATy = ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: UsedArray.size());
1205
1206 auto *GV = new GlobalVariable(M, ATy, false, GlobalValue::AppendingLinkage,
1207 ConstantArray::get(T: ATy, V: UsedArray), Name);
1208
1209 GV->setSection("llvm.metadata");
1210}
1211
1212GlobalVariable *
1213OpenMPIRBuilder::emitKernelExecutionMode(StringRef KernelName,
1214 OMPTgtExecModeFlags Mode) {
1215 auto *Int8Ty = Builder.getInt8Ty();
1216 auto *GVMode = new GlobalVariable(
1217 M, Int8Ty, /*isConstant=*/true, GlobalValue::WeakAnyLinkage,
1218 ConstantInt::get(Ty: Int8Ty, V: Mode), Twine(KernelName, "_exec_mode"));
1219 GVMode->setVisibility(GlobalVariable::ProtectedVisibility);
1220 return GVMode;
1221}
1222
1223Constant *OpenMPIRBuilder::getOrCreateIdent(Constant *SrcLocStr,
1224 uint32_t SrcLocStrSize,
1225 IdentFlag LocFlags,
1226 unsigned Reserve2Flags) {
1227 // Enable "C-mode".
1228 LocFlags |= OMP_IDENT_FLAG_KMPC;
1229
1230 Constant *&Ident =
1231 IdentMap[{SrcLocStr, uint64_t(LocFlags) << 31 | Reserve2Flags}];
1232 if (!Ident) {
1233 Constant *I32Null = ConstantInt::getNullValue(Ty: Int32);
1234 Constant *IdentData[] = {I32Null,
1235 ConstantInt::get(Ty: Int32, V: uint32_t(LocFlags)),
1236 ConstantInt::get(Ty: Int32, V: Reserve2Flags),
1237 ConstantInt::get(Ty: Int32, V: SrcLocStrSize), SrcLocStr};
1238
1239 size_t SrcLocStrArgIdx = 4;
1240 if (OpenMPIRBuilder::Ident->getElementType(N: SrcLocStrArgIdx)
1241 ->getPointerAddressSpace() !=
1242 IdentData[SrcLocStrArgIdx]->getType()->getPointerAddressSpace())
1243 IdentData[SrcLocStrArgIdx] = ConstantExpr::getAddrSpaceCast(
1244 C: SrcLocStr, Ty: OpenMPIRBuilder::Ident->getElementType(N: SrcLocStrArgIdx));
1245 Constant *Initializer =
1246 ConstantStruct::get(T: OpenMPIRBuilder::Ident, V: IdentData);
1247
1248 // Look for existing encoding of the location + flags, not needed but
1249 // minimizes the difference to the existing solution while we transition.
1250 for (GlobalVariable &GV : M.globals())
1251 if (GV.getValueType() == OpenMPIRBuilder::Ident && GV.hasInitializer())
1252 if (GV.getInitializer() == Initializer)
1253 Ident = &GV;
1254
1255 if (!Ident) {
1256 auto *GV = new GlobalVariable(
1257 M, OpenMPIRBuilder::Ident,
1258 /* isConstant = */ true, GlobalValue::PrivateLinkage, Initializer, "",
1259 nullptr, GlobalValue::NotThreadLocal,
1260 M.getDataLayout().getDefaultGlobalsAddressSpace());
1261 GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
1262 GV->setAlignment(Align(8));
1263 Ident = GV;
1264 }
1265 }
1266
1267 return ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: Ident, Ty: IdentPtr);
1268}
1269
1270Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(StringRef LocStr,
1271 uint32_t &SrcLocStrSize) {
1272 SrcLocStrSize = LocStr.size();
1273 Constant *&SrcLocStr = SrcLocStrMap[LocStr];
1274 if (!SrcLocStr) {
1275 Constant *Initializer =
1276 ConstantDataArray::getString(Context&: M.getContext(), Initializer: LocStr);
1277
1278 // Look for existing encoding of the location, not needed but minimizes the
1279 // difference to the existing solution while we transition.
1280 for (GlobalVariable &GV : M.globals())
1281 if (GV.isConstant() && GV.hasInitializer() &&
1282 GV.getInitializer() == Initializer)
1283 return SrcLocStr = ConstantExpr::getPointerCast(C: &GV, Ty: Int8Ptr);
1284
1285 SrcLocStr = Builder.CreateGlobalString(
1286 Str: LocStr, /*Name=*/"", AddressSpace: M.getDataLayout().getDefaultGlobalsAddressSpace(),
1287 M: &M);
1288 }
1289 return SrcLocStr;
1290}
1291
1292Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(StringRef FunctionName,
1293 StringRef FileName,
1294 unsigned Line, unsigned Column,
1295 uint32_t &SrcLocStrSize) {
1296 SmallString<128> Buffer;
1297 Buffer.push_back(Elt: ';');
1298 Buffer.append(RHS: FileName);
1299 Buffer.push_back(Elt: ';');
1300 Buffer.append(RHS: FunctionName);
1301 Buffer.push_back(Elt: ';');
1302 Buffer.append(RHS: std::to_string(val: Line));
1303 Buffer.push_back(Elt: ';');
1304 Buffer.append(RHS: std::to_string(val: Column));
1305 Buffer.push_back(Elt: ';');
1306 Buffer.push_back(Elt: ';');
1307 return getOrCreateSrcLocStr(LocStr: Buffer.str(), SrcLocStrSize);
1308}
1309
1310Constant *
1311OpenMPIRBuilder::getOrCreateDefaultSrcLocStr(uint32_t &SrcLocStrSize) {
1312 StringRef UnknownLoc = ";unknown;unknown;0;0;;";
1313 return getOrCreateSrcLocStr(LocStr: UnknownLoc, SrcLocStrSize);
1314}
1315
1316Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(DebugLoc DL,
1317 uint32_t &SrcLocStrSize,
1318 Function *F) {
1319 DILocation *DIL = DL.get();
1320 if (!DIL)
1321 return getOrCreateDefaultSrcLocStr(SrcLocStrSize);
1322 StringRef FileName =
1323 !DIL->getFilename().empty() ? DIL->getFilename() : M.getName();
1324 StringRef Function = DIL->getScope()->getSubprogram()->getName();
1325 if (Function.empty() && F)
1326 Function = F->getName();
1327 return getOrCreateSrcLocStr(FunctionName: Function, FileName, Line: DIL->getLine(),
1328 Column: DIL->getColumn(), SrcLocStrSize);
1329}
1330
1331Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(const LocationDescription &Loc,
1332 uint32_t &SrcLocStrSize) {
1333 return getOrCreateSrcLocStr(DL: Loc.DL, SrcLocStrSize,
1334 F: Loc.IP.getBlock()->getParent());
1335}
1336
1337Value *OpenMPIRBuilder::getOrCreateThreadID(Value *Ident) {
1338 return createRuntimeFunctionCall(
1339 Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_global_thread_num), Args: Ident,
1340 Name: "omp_global_thread_num");
1341}
1342
1343OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTargetInReduction(
1344 const LocationDescription &Loc, ArrayRef<Value *> OrigPtrs,
1345 ArrayRef<Type *> ResultPtrTys,
1346 function_ref<void(unsigned, Value *)> MapPrivateCB) {
1347 assert(OrigPtrs.size() == ResultPtrTys.size() &&
1348 "expected one result pointer type per in_reduction item");
1349 if (!updateToLocation(Loc))
1350 return Loc.IP;
1351 if (OrigPtrs.empty())
1352 return Builder.saveIP();
1353
1354 // Compute the executing thread's gtid once for the whole target body and
1355 // reuse it for every in_reduction lookup, so a target with several
1356 // in_reduction items does not emit a redundant __kmpc_global_thread_num per
1357 // item.
1358 uint32_t SrcLocStrSize;
1359 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1360 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
1361 Value *Gtid = getOrCreateThreadID(Ident);
1362
1363 // The runtime entry point takes (and returns) a generic, default-address-
1364 // space `ptr`. A NULL descriptor makes the runtime walk the enclosing
1365 // taskgroups to find the matching task_reduction registration for the item.
1366 Type *PtrTy = PointerType::getUnqual(C&: M.getContext());
1367 Value *NullDesc = ConstantPointerNull::get(T: PtrTy);
1368 FunctionCallee GetThData =
1369 getOrCreateRuntimeFunction(M, FnID: OMPRTL___kmpc_task_reduction_get_th_data);
1370
1371 for (unsigned Idx = 0; Idx < OrigPtrs.size(); ++Idx) {
1372 // Normalize a non-default-address-space original pointer to the generic
1373 // address space before the call.
1374 Value *OrigPtr = OrigPtrs[Idx];
1375 if (auto *OrigPtrTy = dyn_cast<PointerType>(Val: OrigPtr->getType());
1376 OrigPtrTy && OrigPtrTy->getAddressSpace() != 0)
1377 OrigPtr = Builder.CreateAddrSpaceCast(V: OrigPtr, DestTy: PtrTy);
1378
1379 Value *Priv = Builder.CreateCall(Callee: GetThData, Args: {Gtid, NullDesc, OrigPtr},
1380 Name: "omp.inred.priv");
1381
1382 // Cast the returned private pointer back to the requested address space
1383 // when it differs.
1384 if (auto *ResPtrTy = dyn_cast<PointerType>(Val: ResultPtrTys[Idx]);
1385 ResPtrTy && ResPtrTy->getAddressSpace() != 0)
1386 Priv = Builder.CreateAddrSpaceCast(V: Priv, DestTy: ResultPtrTys[Idx]);
1387
1388 MapPrivateCB(Idx, Priv);
1389 }
1390 return Builder.saveIP();
1391}
1392
1393OpenMPIRBuilder::InsertPointOrErrorTy
1394OpenMPIRBuilder::createBarrier(const LocationDescription &Loc, Directive Kind,
1395 bool ForceSimpleCall, bool CheckCancelFlag) {
1396 if (!updateToLocation(Loc))
1397 return Loc.IP;
1398
1399 // Build call __kmpc_cancel_barrier(loc, thread_id) or
1400 // __kmpc_barrier(loc, thread_id);
1401
1402 IdentFlag BarrierLocFlags;
1403 switch (Kind) {
1404 case OMPD_for:
1405 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_FOR;
1406 break;
1407 case OMPD_sections:
1408 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_SECTIONS;
1409 break;
1410 case OMPD_single:
1411 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_SINGLE;
1412 break;
1413 case OMPD_barrier:
1414 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_EXPL;
1415 break;
1416 default:
1417 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL;
1418 break;
1419 }
1420
1421 uint32_t SrcLocStrSize;
1422 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1423 Value *Args[] = {
1424 getOrCreateIdent(SrcLocStr, SrcLocStrSize, LocFlags: BarrierLocFlags),
1425 getOrCreateThreadID(Ident: getOrCreateIdent(SrcLocStr, SrcLocStrSize))};
1426
1427 // If we are in a cancellable parallel region, barriers are cancellation
1428 // points.
1429 // TODO: Check why we would force simple calls or to ignore the cancel flag.
1430 bool UseCancelBarrier =
1431 !ForceSimpleCall && isLastFinalizationInfoCancellable(DK: OMPD_parallel);
1432
1433 Value *Result = createRuntimeFunctionCall(
1434 Callee: getOrCreateRuntimeFunctionPtr(FnID: UseCancelBarrier
1435 ? OMPRTL___kmpc_cancel_barrier
1436 : OMPRTL___kmpc_barrier),
1437 Args);
1438
1439 if (UseCancelBarrier && CheckCancelFlag)
1440 if (Error Err = emitCancelationCheckImpl(CancelFlag: Result, CanceledDirective: OMPD_parallel))
1441 return Err;
1442
1443 return Builder.saveIP();
1444}
1445
1446OpenMPIRBuilder::InsertPointOrErrorTy
1447OpenMPIRBuilder::createCancel(const LocationDescription &Loc,
1448 Value *IfCondition,
1449 omp::Directive CanceledDirective) {
1450 if (!updateToLocation(Loc))
1451 return Loc.IP;
1452
1453 // LLVM utilities like blocks with terminators.
1454 auto *UI = Builder.CreateUnreachable();
1455
1456 Instruction *ThenTI = UI, *ElseTI = nullptr;
1457 if (IfCondition) {
1458 SplitBlockAndInsertIfThenElse(Cond: IfCondition, SplitBefore: UI, ThenTerm: &ThenTI, ElseTerm: &ElseTI);
1459
1460 // Even if the if condition evaluates to false, this should count as a
1461 // cancellation point
1462 Builder.SetInsertPoint(ElseTI);
1463 auto ElseIP = Builder.saveIP();
1464
1465 InsertPointOrErrorTy IPOrErr = createCancellationPoint(
1466 Loc: LocationDescription{ElseIP, Loc.DL}, CanceledDirective);
1467 if (!IPOrErr)
1468 return IPOrErr;
1469 }
1470
1471 Builder.SetInsertPoint(ThenTI);
1472
1473 Value *CancelKind = nullptr;
1474 switch (CanceledDirective) {
1475#define OMP_CANCEL_KIND(Enum, Str, DirectiveEnum, Value) \
1476 case DirectiveEnum: \
1477 CancelKind = Builder.getInt32(Value); \
1478 break;
1479#include "llvm/Frontend/OpenMP/OMPKinds.def"
1480 default:
1481 llvm_unreachable("Unknown cancel kind!");
1482 }
1483
1484 uint32_t SrcLocStrSize;
1485 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1486 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
1487 Value *Args[] = {Ident, getOrCreateThreadID(Ident), CancelKind};
1488 Value *Result = createRuntimeFunctionCall(
1489 Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_cancel), Args);
1490
1491 // The actual cancel logic is shared with others, e.g., cancel_barriers.
1492 if (Error Err = emitCancelationCheckImpl(CancelFlag: Result, CanceledDirective))
1493 return Err;
1494
1495 // Update the insertion point and remove the terminator we introduced.
1496 Builder.SetInsertPoint(UI->getParent());
1497 UI->eraseFromParent();
1498
1499 return Builder.saveIP();
1500}
1501
1502OpenMPIRBuilder::InsertPointOrErrorTy
1503OpenMPIRBuilder::createCancellationPoint(const LocationDescription &Loc,
1504 omp::Directive CanceledDirective) {
1505 if (!updateToLocation(Loc))
1506 return Loc.IP;
1507
1508 // LLVM utilities like blocks with terminators.
1509 auto *UI = Builder.CreateUnreachable();
1510 Builder.SetInsertPoint(UI);
1511
1512 Value *CancelKind = nullptr;
1513 switch (CanceledDirective) {
1514#define OMP_CANCEL_KIND(Enum, Str, DirectiveEnum, Value) \
1515 case DirectiveEnum: \
1516 CancelKind = Builder.getInt32(Value); \
1517 break;
1518#include "llvm/Frontend/OpenMP/OMPKinds.def"
1519 default:
1520 llvm_unreachable("Unknown cancel kind!");
1521 }
1522
1523 uint32_t SrcLocStrSize;
1524 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1525 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
1526 Value *Args[] = {Ident, getOrCreateThreadID(Ident), CancelKind};
1527 Value *Result = createRuntimeFunctionCall(
1528 Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_cancellationpoint), Args);
1529
1530 // The actual cancel logic is shared with others, e.g., cancel_barriers.
1531 if (Error Err = emitCancelationCheckImpl(CancelFlag: Result, CanceledDirective))
1532 return Err;
1533
1534 // Update the insertion point and remove the terminator we introduced.
1535 Builder.SetInsertPoint(UI->getParent());
1536 UI->eraseFromParent();
1537
1538 return Builder.saveIP();
1539}
1540
1541OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitTargetKernel(
1542 const LocationDescription &Loc, InsertPointTy AllocaIP, Value *&Return,
1543 Value *Ident, Value *DeviceID, Value *NumTeams, Value *NumThreads,
1544 Value *HostPtr, ArrayRef<Value *> KernelArgs) {
1545 if (!updateToLocation(Loc))
1546 return Loc.IP;
1547
1548 Builder.restoreIP(IP: AllocaIP);
1549 auto *KernelArgsPtr =
1550 Builder.CreateAlloca(Ty: OpenMPIRBuilder::KernelArgs, ArraySize: nullptr, Name: "kernel_args");
1551 updateToLocation(Loc);
1552
1553 for (unsigned I = 0, Size = KernelArgs.size(); I != Size; ++I) {
1554 llvm::Value *Arg =
1555 Builder.CreateStructGEP(Ty: OpenMPIRBuilder::KernelArgs, Ptr: KernelArgsPtr, Idx: I);
1556 Builder.CreateAlignedStore(
1557 Val: KernelArgs[I], Ptr: Arg,
1558 Align: M.getDataLayout().getPrefTypeAlign(Ty: KernelArgs[I]->getType()));
1559 }
1560
1561 SmallVector<Value *> OffloadingArgs{Ident, DeviceID, NumTeams,
1562 NumThreads, HostPtr, KernelArgsPtr};
1563
1564 Return = createRuntimeFunctionCall(
1565 Callee: getOrCreateRuntimeFunction(M, FnID: OMPRTL___tgt_target_kernel),
1566 Args: OffloadingArgs);
1567
1568 return Builder.saveIP();
1569}
1570
1571OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitKernelLaunch(
1572 const LocationDescription &Loc, Value *OutlinedFnID,
1573 EmitFallbackCallbackTy EmitTargetCallFallbackCB, TargetKernelArgs &Args,
1574 Value *DeviceID, Value *RTLoc, InsertPointTy AllocaIP) {
1575
1576 if (!updateToLocation(Loc))
1577 return Loc.IP;
1578
1579 // On top of the arrays that were filled up, the target offloading call
1580 // takes as arguments the device id as well as the host pointer. The host
1581 // pointer is used by the runtime library to identify the current target
1582 // region, so it only has to be unique and not necessarily point to
1583 // anything. It could be the pointer to the outlined function that
1584 // implements the target region, but we aren't using that so that the
1585 // compiler doesn't need to keep that, and could therefore inline the host
1586 // function if proven worthwhile during optimization.
1587
1588 // From this point on, we need to have an ID of the target region defined.
1589 assert(OutlinedFnID && "Invalid outlined function ID!");
1590 (void)OutlinedFnID;
1591
1592 // Return value of the runtime offloading call.
1593 Value *Return = nullptr;
1594
1595 // Arguments for the target kernel.
1596 SmallVector<Value *> ArgsVector;
1597 getKernelArgsVector(KernelArgs&: Args, Builder, ArgsVector);
1598
1599 // The target region is an outlined function launched by the runtime
1600 // via calls to __tgt_target_kernel().
1601 //
1602 // Note that on the host and CPU targets, the runtime implementation of
1603 // these calls simply call the outlined function without forking threads.
1604 // The outlined functions themselves have runtime calls to
1605 // __kmpc_fork_teams() and __kmpc_fork() for this purpose, codegen'd by
1606 // the compiler in emitTeamsCall() and emitParallelCall().
1607 //
1608 // In contrast, on the NVPTX target, the implementation of
1609 // __tgt_target_teams() launches a GPU kernel with the requested number
1610 // of teams and threads so no additional calls to the runtime are required.
1611 // Check the error code and execute the host version if required.
1612 Builder.restoreIP(IP: emitTargetKernel(
1613 Loc: Builder, AllocaIP, Return, Ident: RTLoc, DeviceID, NumTeams: Args.NumTeams.front(),
1614 NumThreads: Args.NumThreads.front(), HostPtr: OutlinedFnID, KernelArgs: ArgsVector));
1615
1616 BasicBlock *OffloadFailedBlock =
1617 BasicBlock::Create(Context&: Builder.getContext(), Name: "omp_offload.failed");
1618 BasicBlock *OffloadContBlock =
1619 BasicBlock::Create(Context&: Builder.getContext(), Name: "omp_offload.cont");
1620 Value *Failed = Builder.CreateIsNotNull(Arg: Return);
1621 Builder.CreateCondBr(Cond: Failed, True: OffloadFailedBlock, False: OffloadContBlock);
1622
1623 auto CurFn = Builder.GetInsertBlock()->getParent();
1624 emitBlock(BB: OffloadFailedBlock, CurFn);
1625 InsertPointOrErrorTy AfterIP = EmitTargetCallFallbackCB(Builder.saveIP());
1626 if (!AfterIP)
1627 return AfterIP.takeError();
1628 Builder.restoreIP(IP: *AfterIP);
1629 emitBranch(Target: OffloadContBlock);
1630 emitBlock(BB: OffloadContBlock, CurFn, /*IsFinished=*/true);
1631 return Builder.saveIP();
1632}
1633
1634Error OpenMPIRBuilder::emitCancelationCheckImpl(
1635 Value *CancelFlag, omp::Directive CanceledDirective) {
1636 assert(isLastFinalizationInfoCancellable(CanceledDirective) &&
1637 "Unexpected cancellation!");
1638
1639 // For a cancel barrier we create two new blocks.
1640 BasicBlock *BB = Builder.GetInsertBlock();
1641 BasicBlock *NonCancellationBlock;
1642 if (Builder.GetInsertPoint() == BB->end()) {
1643 // TODO: This branch will not be needed once we moved to the
1644 // OpenMPIRBuilder codegen completely.
1645 NonCancellationBlock = BasicBlock::Create(
1646 Context&: BB->getContext(), Name: BB->getName() + ".cont", Parent: BB->getParent());
1647 } else {
1648 NonCancellationBlock = SplitBlock(Old: BB, SplitPt: &*Builder.GetInsertPoint());
1649 BB->getTerminator()->eraseFromParent();
1650 Builder.SetInsertPoint(BB);
1651 }
1652 BasicBlock *CancellationBlock = BasicBlock::Create(
1653 Context&: BB->getContext(), Name: BB->getName() + ".cncl", Parent: BB->getParent());
1654
1655 // Jump to them based on the return value.
1656 Value *Cmp = Builder.CreateIsNull(Arg: CancelFlag);
1657 Builder.CreateCondBr(Cond: Cmp, True: NonCancellationBlock, False: CancellationBlock,
1658 /* TODO weight */ BranchWeights: nullptr, Unpredictable: nullptr);
1659
1660 // From the cancellation block we finalize all variables and go to the
1661 // post finalization block that is known to the FiniCB callback.
1662 auto &FI = FinalizationStack.back();
1663 Expected<BasicBlock *> FiniBBOrErr = FI.getFiniBB(Builder);
1664 if (!FiniBBOrErr)
1665 return FiniBBOrErr.takeError();
1666 Builder.SetInsertPoint(CancellationBlock);
1667 Builder.CreateBr(Dest: *FiniBBOrErr);
1668
1669 // The continuation block is where code generation continues.
1670 Builder.SetInsertPoint(TheBB: NonCancellationBlock, IP: NonCancellationBlock->begin());
1671 return Error::success();
1672}
1673
1674/// Create wrapper function used to gather the outlined function's argument
1675/// structure from a shared buffer and to forward them to it when running in
1676/// Generic mode.
1677///
1678/// The outlined function is expected to receive 2 integer arguments followed by
1679/// an optional pointer argument to an argument structure holding the rest.
1680static Function *createTargetParallelWrapper(OpenMPIRBuilder *OMPIRBuilder,
1681 Function &OutlinedFn) {
1682 size_t NumArgs = OutlinedFn.arg_size();
1683 assert((NumArgs == 2 || NumArgs == 3) &&
1684 "expected a 2-3 argument parallel outlined function");
1685 bool UseArgStruct = NumArgs == 3;
1686
1687 IRBuilder<> &Builder = OMPIRBuilder->Builder;
1688 IRBuilder<>::InsertPointGuard IPG(Builder);
1689 auto *FnTy = FunctionType::get(Result: Builder.getVoidTy(),
1690 Params: {Builder.getInt16Ty(), Builder.getInt32Ty()},
1691 /*isVarArg=*/false);
1692 auto *WrapperFn =
1693 Function::Create(Ty: FnTy, Linkage: GlobalValue::InternalLinkage,
1694 N: OutlinedFn.getName() + ".wrapper", M&: OMPIRBuilder->M);
1695
1696 WrapperFn->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
1697 WrapperFn->addParamAttr(ArgNo: 0, Kind: Attribute::ZExt);
1698 WrapperFn->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
1699
1700 BasicBlock *EntryBB =
1701 BasicBlock::Create(Context&: OMPIRBuilder->M.getContext(), Name: "entry", Parent: WrapperFn);
1702 Builder.SetInsertPoint(EntryBB);
1703
1704 // Allocation.
1705 Value *AddrAlloca = Builder.CreateAlloca(Ty: Builder.getInt32Ty(),
1706 /*ArraySize=*/nullptr, Name: "addr");
1707 AddrAlloca = Builder.CreatePointerBitCastOrAddrSpaceCast(
1708 V: AddrAlloca, DestTy: Builder.getPtrTy(/*AddrSpace=*/0),
1709 Name: AddrAlloca->getName() + ".ascast");
1710
1711 Value *ZeroAlloca = Builder.CreateAlloca(Ty: Builder.getInt32Ty(),
1712 /*ArraySize=*/nullptr, Name: "zero");
1713 ZeroAlloca = Builder.CreatePointerBitCastOrAddrSpaceCast(
1714 V: ZeroAlloca, DestTy: Builder.getPtrTy(/*AddrSpace=*/0),
1715 Name: ZeroAlloca->getName() + ".ascast");
1716
1717 Value *ArgsAlloca = nullptr;
1718 if (UseArgStruct) {
1719 ArgsAlloca = Builder.CreateAlloca(Ty: Builder.getPtrTy(),
1720 /*ArraySize=*/nullptr, Name: "global_args");
1721 ArgsAlloca = Builder.CreatePointerBitCastOrAddrSpaceCast(
1722 V: ArgsAlloca, DestTy: Builder.getPtrTy(/*AddrSpace=*/0),
1723 Name: ArgsAlloca->getName() + ".ascast");
1724 }
1725
1726 // Initialization.
1727 Builder.CreateStore(Val: WrapperFn->getArg(i: 1), Ptr: AddrAlloca);
1728 Builder.CreateStore(Val: Builder.getInt32(C: 0), Ptr: ZeroAlloca);
1729 if (UseArgStruct) {
1730 Builder.CreateCall(
1731 Callee: OMPIRBuilder->getOrCreateRuntimeFunctionPtr(
1732 FnID: llvm::omp::RuntimeFunction::OMPRTL___kmpc_get_shared_variables),
1733 Args: {ArgsAlloca});
1734 }
1735
1736 SmallVector<Value *, 3> Args{AddrAlloca, ZeroAlloca};
1737
1738 // Load structArg from global_args.
1739 if (UseArgStruct) {
1740 Value *StructArg = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ArgsAlloca);
1741 StructArg = Builder.CreateInBoundsGEP(Ty: Builder.getPtrTy(), Ptr: StructArg,
1742 IdxList: {Builder.getInt64(C: 0)});
1743 StructArg = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: StructArg, Name: "structArg");
1744 Args.push_back(Elt: StructArg);
1745 }
1746
1747 // Call the outlined function holding the parallel body.
1748 Builder.CreateCall(Callee: &OutlinedFn, Args);
1749 Builder.CreateRetVoid();
1750
1751 return WrapperFn;
1752}
1753
1754// Callback used to create OpenMP runtime calls to support
1755// omp parallel clause for the device.
1756// We need to use this callback to replace call to the OutlinedFn in OuterFn
1757// by the call to the OpenMP DeviceRTL runtime function (kmpc_parallel_60)
1758static void targetParallelCallback(
1759 OpenMPIRBuilder *OMPIRBuilder, Function &OutlinedFn, Function *OuterFn,
1760 BasicBlock *OuterAllocaBB, Value *Ident, Value *IfCondition,
1761 Value *NumThreads, Instruction *PrivTID, AllocaInst *PrivTIDAddr,
1762 Value *ThreadID, const SmallVector<Instruction *, 4> &ToBeDeleted) {
1763 assert(OutlinedFn.arg_size() >= 2 &&
1764 "Expected at least tid and bounded tid as arguments");
1765 unsigned NumCapturedVars = OutlinedFn.arg_size() - /* tid & bounded tid */ 2;
1766
1767 // Add some known attributes.
1768 IRBuilder<> &Builder = OMPIRBuilder->Builder;
1769 OutlinedFn.addParamAttr(ArgNo: 0, Kind: Attribute::NoAlias);
1770 OutlinedFn.addParamAttr(ArgNo: 1, Kind: Attribute::NoAlias);
1771 OutlinedFn.addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
1772 OutlinedFn.addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
1773 OutlinedFn.addFnAttr(Kind: Attribute::NoUnwind);
1774
1775 CallInst *CI = cast<CallInst>(Val: OutlinedFn.user_back());
1776 assert(CI && "Expected call instruction to outlined function");
1777 CI->getParent()->setName("omp_parallel");
1778
1779 Builder.SetInsertPoint(CI);
1780 Type *PtrTy = OMPIRBuilder->VoidPtr;
1781
1782 // Add alloca for kernel args
1783 OpenMPIRBuilder ::InsertPointTy CurrentIP = Builder.saveIP();
1784 Builder.SetInsertPoint(TheBB: OuterAllocaBB, IP: OuterAllocaBB->getFirstInsertionPt());
1785 AllocaInst *ArgsAlloca =
1786 Builder.CreateAlloca(Ty: ArrayType::get(ElementType: PtrTy, NumElements: NumCapturedVars));
1787 Value *Args = ArgsAlloca;
1788 // Add address space cast if array for storing arguments is not allocated
1789 // in address space 0
1790 if (ArgsAlloca->getAddressSpace())
1791 Args = Builder.CreatePointerCast(V: ArgsAlloca, DestTy: PtrTy);
1792 Builder.restoreIP(IP: CurrentIP);
1793
1794 // Store captured vars which are used by kmpc_parallel_60
1795 for (unsigned Idx = 0; Idx < NumCapturedVars; Idx++) {
1796 Value *V = *(CI->arg_begin() + 2 + Idx);
1797 Value *StoreAddress = Builder.CreateConstInBoundsGEP2_64(
1798 Ty: ArrayType::get(ElementType: PtrTy, NumElements: NumCapturedVars), Ptr: Args, Idx0: 0, Idx1: Idx);
1799 Builder.CreateStore(Val: V, Ptr: StoreAddress);
1800 }
1801
1802 Value *Cond =
1803 IfCondition ? Builder.CreateSExtOrTrunc(V: IfCondition, DestTy: OMPIRBuilder->Int32)
1804 : Builder.getInt32(C: 1);
1805 Value *NumThreadsArg =
1806 NumThreads ? Builder.CreateZExtOrTrunc(V: NumThreads, DestTy: OMPIRBuilder->Int32)
1807 : Builder.getInt32(C: -1);
1808
1809 // If this is not a Generic kernel, we can skip generating the wrapper.
1810 Value *WrapperFn;
1811 if (isGenericKernel(Fn&: *OuterFn))
1812 WrapperFn = createTargetParallelWrapper(OMPIRBuilder, OutlinedFn);
1813 else
1814 WrapperFn = Constant::getNullValue(Ty: PtrTy);
1815
1816 // Build kmpc_parallel_60 call
1817 Value *Parallel60CallArgs[] = {
1818 /* identifier*/ Ident,
1819 /* global thread num*/ ThreadID,
1820 /* if expression */ Cond,
1821 /* number of threads */ NumThreadsArg,
1822 /* Proc bind */ Builder.getInt32(C: -1),
1823 /* outlined function */ &OutlinedFn,
1824 /* wrapper function */ WrapperFn,
1825 /* arguments of the outlined funciton*/ Args,
1826 /* number of arguments */ Builder.getInt64(C: NumCapturedVars),
1827 /* strict for number of threads */ Builder.getInt32(C: 0)};
1828
1829 FunctionCallee RTLFn =
1830 OMPIRBuilder->getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_parallel_60);
1831
1832 OMPIRBuilder->createRuntimeFunctionCall(Callee: RTLFn, Args: Parallel60CallArgs);
1833
1834 LLVM_DEBUG(dbgs() << "With kmpc_parallel_60 placed: "
1835 << *Builder.GetInsertBlock()->getParent() << "\n");
1836
1837 // Initialize the local TID stack location with the argument value.
1838 Builder.SetInsertPoint(PrivTID);
1839 Function::arg_iterator OutlinedAI = OutlinedFn.arg_begin();
1840 Builder.CreateStore(Val: Builder.CreateLoad(Ty: OMPIRBuilder->Int32, Ptr: OutlinedAI),
1841 Ptr: PrivTIDAddr);
1842
1843 // Remove redundant call to the outlined function.
1844 CI->eraseFromParent();
1845
1846 for (Instruction *I : ToBeDeleted) {
1847 I->eraseFromParent();
1848 }
1849}
1850
1851// Callback used to create OpenMP runtime calls to support
1852// omp parallel clause for the host.
1853// We need to use this callback to replace call to the OutlinedFn in OuterFn
1854// by the call to the OpenMP host runtime function ( __kmpc_fork_call[_if])
1855static void
1856hostParallelCallback(OpenMPIRBuilder *OMPIRBuilder, Function &OutlinedFn,
1857 Function *OuterFn, Value *Ident, Value *IfCondition,
1858 Instruction *PrivTID, AllocaInst *PrivTIDAddr,
1859 const SmallVector<Instruction *, 4> &ToBeDeleted) {
1860 IRBuilder<> &Builder = OMPIRBuilder->Builder;
1861 FunctionCallee RTLFn;
1862 if (IfCondition) {
1863 RTLFn =
1864 OMPIRBuilder->getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_fork_call_if);
1865 } else {
1866 RTLFn =
1867 OMPIRBuilder->getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_fork_call);
1868 }
1869 if (auto *F = dyn_cast<Function>(Val: RTLFn.getCallee())) {
1870 if (!F->hasMetadata(KindID: LLVMContext::MD_callback)) {
1871 LLVMContext &Ctx = F->getContext();
1872 MDBuilder MDB(Ctx);
1873 // Annotate the callback behavior of the __kmpc_fork_call:
1874 // - The callback callee is argument number 2 (microtask).
1875 // - The first two arguments of the callback callee are unknown (-1).
1876 // - All variadic arguments to the __kmpc_fork_call are passed to the
1877 // callback callee.
1878 F->addMetadata(KindID: LLVMContext::MD_callback,
1879 MD&: *MDNode::get(Context&: Ctx, MDs: {MDB.createCallbackEncoding(
1880 CalleeArgNo: 2, Arguments: {-1, -1},
1881 /* VarArgsArePassed */ true)}));
1882 }
1883 }
1884 // Add some known attributes.
1885 OutlinedFn.addParamAttr(ArgNo: 0, Kind: Attribute::NoAlias);
1886 OutlinedFn.addParamAttr(ArgNo: 1, Kind: Attribute::NoAlias);
1887 OutlinedFn.addFnAttr(Kind: Attribute::NoUnwind);
1888
1889 assert(OutlinedFn.arg_size() >= 2 &&
1890 "Expected at least tid and bounded tid as arguments");
1891 unsigned NumCapturedVars = OutlinedFn.arg_size() - /* tid & bounded tid */ 2;
1892
1893 CallInst *CI = cast<CallInst>(Val: OutlinedFn.user_back());
1894 CI->getParent()->setName("omp_parallel");
1895 Builder.SetInsertPoint(CI);
1896
1897 // Build call __kmpc_fork_call[_if](Ident, n, microtask, var1, .., varn);
1898 Value *ForkCallArgs[] = {Ident, Builder.getInt32(C: NumCapturedVars),
1899 &OutlinedFn};
1900
1901 SmallVector<Value *, 16> RealArgs;
1902 RealArgs.append(in_start: std::begin(arr&: ForkCallArgs), in_end: std::end(arr&: ForkCallArgs));
1903 if (IfCondition) {
1904 Value *Cond = Builder.CreateSExtOrTrunc(V: IfCondition, DestTy: OMPIRBuilder->Int32);
1905 RealArgs.push_back(Elt: Cond);
1906 }
1907 RealArgs.append(in_start: CI->arg_begin() + /* tid & bound tid */ 2, in_end: CI->arg_end());
1908
1909 // __kmpc_fork_call_if always expects a void ptr as the last argument
1910 // If there are no arguments, pass a null pointer.
1911 auto PtrTy = OMPIRBuilder->VoidPtr;
1912 if (IfCondition && NumCapturedVars == 0) {
1913 Value *NullPtrValue = Constant::getNullValue(Ty: PtrTy);
1914 RealArgs.push_back(Elt: NullPtrValue);
1915 }
1916
1917 OMPIRBuilder->createRuntimeFunctionCall(Callee: RTLFn, Args: RealArgs);
1918
1919 LLVM_DEBUG(dbgs() << "With fork_call placed: "
1920 << *Builder.GetInsertBlock()->getParent() << "\n");
1921
1922 // Initialize the local TID stack location with the argument value.
1923 Builder.SetInsertPoint(PrivTID);
1924 Function::arg_iterator OutlinedAI = OutlinedFn.arg_begin();
1925 Builder.CreateStore(Val: Builder.CreateLoad(Ty: OMPIRBuilder->Int32, Ptr: OutlinedAI),
1926 Ptr: PrivTIDAddr);
1927
1928 // Remove redundant call to the outlined function.
1929 CI->eraseFromParent();
1930
1931 for (Instruction *I : ToBeDeleted) {
1932 I->eraseFromParent();
1933 }
1934}
1935
1936OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createParallel(
1937 const LocationDescription &Loc, InsertPointTy OuterAllocIP,
1938 ArrayRef<BasicBlock *> OuterDeallocBlocks, BodyGenCallbackTy BodyGenCB,
1939 PrivatizeCallbackTy PrivCB, FinalizeCallbackTy FiniCB, Value *IfCondition,
1940 Value *NumThreads, omp::ProcBindKind ProcBind, bool IsCancellable) {
1941 assert(!isConflictIP(Loc.IP, OuterAllocIP) && "IPs must not be ambiguous");
1942
1943 if (!updateToLocation(Loc))
1944 return Loc.IP;
1945
1946 uint32_t SrcLocStrSize;
1947 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1948 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
1949 const bool NeedThreadID = NumThreads || Config.isTargetDevice() ||
1950 (ProcBind != OMP_PROC_BIND_default);
1951 Value *ThreadID = NeedThreadID ? getOrCreateThreadID(Ident) : nullptr;
1952 // If we generate code for the target device, we need to allocate
1953 // struct for aggregate params in the device default alloca address space.
1954 // OpenMP runtime requires that the params of the extracted functions are
1955 // passed as zero address space pointers. This flag ensures that extracted
1956 // function arguments are declared in zero address space
1957 bool ArgsInZeroAddressSpace = Config.isTargetDevice();
1958
1959 // Build call __kmpc_push_num_threads(&Ident, global_tid, num_threads)
1960 // only if we compile for host side.
1961 if (NumThreads && !Config.isTargetDevice()) {
1962 Value *Args[] = {
1963 Ident, ThreadID,
1964 Builder.CreateIntCast(V: NumThreads, DestTy: Int32, /*isSigned*/ false)};
1965 createRuntimeFunctionCall(
1966 Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_push_num_threads), Args);
1967 }
1968
1969 if (ProcBind != OMP_PROC_BIND_default) {
1970 // Build call __kmpc_push_proc_bind(&Ident, global_tid, proc_bind)
1971 Value *Args[] = {
1972 Ident, ThreadID,
1973 ConstantInt::get(Ty: Int32, V: unsigned(ProcBind), /*isSigned=*/IsSigned: true)};
1974 createRuntimeFunctionCall(
1975 Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_push_proc_bind), Args);
1976 }
1977
1978 BasicBlock *InsertBB = Builder.GetInsertBlock();
1979 Function *OuterFn = InsertBB->getParent();
1980
1981 // Save the outer alloca block because the insertion iterator may get
1982 // invalidated and we still need this later.
1983 BasicBlock *OuterAllocaBlock = OuterAllocIP.getBlock();
1984
1985 // Vector to remember instructions we used only during the modeling but which
1986 // we want to delete at the end.
1987 SmallVector<Instruction *, 4> ToBeDeleted;
1988
1989 // Change the location to the outer alloca insertion point to create and
1990 // initialize the allocas we pass into the parallel region.
1991 InsertPointTy NewOuter(OuterAllocaBlock, OuterAllocaBlock->begin());
1992 Builder.restoreIP(IP: NewOuter);
1993 AllocaInst *TIDAddrAlloca = Builder.CreateAlloca(Ty: Int32, ArraySize: nullptr, Name: "tid.addr");
1994 AllocaInst *ZeroAddrAlloca =
1995 Builder.CreateAlloca(Ty: Int32, ArraySize: nullptr, Name: "zero.addr");
1996 Instruction *TIDAddr = TIDAddrAlloca;
1997 Instruction *ZeroAddr = ZeroAddrAlloca;
1998 if (ArgsInZeroAddressSpace && M.getDataLayout().getAllocaAddrSpace() != 0) {
1999 // Add additional casts to enforce pointers in zero address space
2000 TIDAddr = new AddrSpaceCastInst(
2001 TIDAddrAlloca, PointerType ::get(C&: M.getContext(), AddressSpace: 0), "tid.addr.ascast");
2002 TIDAddr->insertAfter(InsertPos: TIDAddrAlloca->getIterator());
2003 ToBeDeleted.push_back(Elt: TIDAddr);
2004 ZeroAddr = new AddrSpaceCastInst(ZeroAddrAlloca,
2005 PointerType ::get(C&: M.getContext(), AddressSpace: 0),
2006 "zero.addr.ascast");
2007 ZeroAddr->insertAfter(InsertPos: ZeroAddrAlloca->getIterator());
2008 ToBeDeleted.push_back(Elt: ZeroAddr);
2009 }
2010
2011 // We only need TIDAddr and ZeroAddr for modeling purposes to get the
2012 // associated arguments in the outlined function, so we delete them later.
2013 ToBeDeleted.push_back(Elt: TIDAddrAlloca);
2014 ToBeDeleted.push_back(Elt: ZeroAddrAlloca);
2015
2016 // Create an artificial insertion point that will also ensure the blocks we
2017 // are about to split are not degenerated.
2018 auto *UI = new UnreachableInst(Builder.getContext(), InsertBB);
2019
2020 BasicBlock *EntryBB = UI->getParent();
2021 BasicBlock *PRegEntryBB = EntryBB->splitBasicBlock(I: UI, BBName: "omp.par.entry");
2022 BasicBlock *PRegBodyBB = PRegEntryBB->splitBasicBlock(I: UI, BBName: "omp.par.region");
2023 BasicBlock *PRegPreFiniBB =
2024 PRegBodyBB->splitBasicBlock(I: UI, BBName: "omp.par.pre_finalize");
2025 BasicBlock *PRegExitBB = PRegPreFiniBB->splitBasicBlock(I: UI, BBName: "omp.par.exit");
2026
2027 auto FiniCBWrapper = [&](InsertPointTy IP) {
2028 // Hide "open-ended" blocks from the given FiniCB by setting the right jump
2029 // target to the region exit block.
2030 if (IP.getBlock()->end() == IP.getPoint()) {
2031 IRBuilder<>::InsertPointGuard IPG(Builder);
2032 Builder.restoreIP(IP);
2033 Instruction *I = Builder.CreateBr(Dest: PRegExitBB);
2034 IP = InsertPointTy(I->getParent(), I->getIterator());
2035 }
2036 assert(IP.getBlock()->getTerminator()->getNumSuccessors() == 1 &&
2037 IP.getBlock()->getTerminator()->getSuccessor(0) == PRegExitBB &&
2038 "Unexpected insertion point for finalization call!");
2039 return FiniCB(IP);
2040 };
2041
2042 FinalizationStack.push_back(Elt: {FiniCBWrapper, OMPD_parallel, IsCancellable});
2043
2044 // Generate the privatization allocas in the block that will become the entry
2045 // of the outlined function.
2046 Builder.SetInsertPoint(PRegEntryBB->getTerminator());
2047 InsertPointTy InnerAllocaIP = Builder.saveIP();
2048
2049 AllocaInst *PrivTIDAddr =
2050 Builder.CreateAlloca(Ty: Int32, ArraySize: nullptr, Name: "tid.addr.local");
2051 Instruction *PrivTID = Builder.CreateLoad(Ty: Int32, Ptr: PrivTIDAddr, Name: "tid");
2052
2053 // Add some fake uses for OpenMP provided arguments.
2054 ToBeDeleted.push_back(Elt: Builder.CreateLoad(Ty: Int32, Ptr: TIDAddr, Name: "tid.addr.use"));
2055 Instruction *ZeroAddrUse =
2056 Builder.CreateLoad(Ty: Int32, Ptr: ZeroAddr, Name: "zero.addr.use");
2057 ToBeDeleted.push_back(Elt: ZeroAddrUse);
2058
2059 // EntryBB
2060 // |
2061 // V
2062 // PRegionEntryBB <- Privatization allocas are placed here.
2063 // |
2064 // V
2065 // PRegionBodyBB <- BodeGen is invoked here.
2066 // |
2067 // V
2068 // PRegPreFiniBB <- The block we will start finalization from.
2069 // |
2070 // V
2071 // PRegionExitBB <- A common exit to simplify block collection.
2072 //
2073
2074 LLVM_DEBUG(dbgs() << "Before body codegen: " << *OuterFn << "\n");
2075
2076 // Let the caller create the body.
2077 assert(BodyGenCB && "Expected body generation callback!");
2078 InsertPointTy CodeGenIP(PRegBodyBB, PRegBodyBB->begin());
2079 if (Error Err = BodyGenCB(InnerAllocaIP, CodeGenIP, PRegExitBB))
2080 return Err;
2081
2082 LLVM_DEBUG(dbgs() << "After body codegen: " << *OuterFn << "\n");
2083
2084 // If OuterFn is a Generic kernel, we need to use device shared memory to
2085 // allocate argument structures. Otherwise, we use stack allocations as usual.
2086 bool UsesDeviceSharedMemory =
2087 Config.isTargetDevice() && isGenericKernel(Fn&: *OuterFn);
2088 std::unique_ptr<OutlineInfo> OI =
2089 UsesDeviceSharedMemory
2090 ? std::make_unique<DeviceSharedMemOutlineInfo>(args&: *this)
2091 : std::make_unique<OutlineInfo>();
2092
2093 if (Config.isTargetDevice()) {
2094 // Generate OpenMP target specific runtime call
2095 OI->PostOutlineCB = [=, ToBeDeletedVec =
2096 std::move(ToBeDeleted)](Function &OutlinedFn) {
2097 targetParallelCallback(OMPIRBuilder: this, OutlinedFn, OuterFn, OuterAllocaBB: OuterAllocaBlock, Ident,
2098 IfCondition, NumThreads, PrivTID, PrivTIDAddr,
2099 ThreadID, ToBeDeleted: ToBeDeletedVec);
2100 };
2101 } else {
2102 // Generate OpenMP host runtime call
2103 OI->PostOutlineCB = [=, ToBeDeletedVec =
2104 std::move(ToBeDeleted)](Function &OutlinedFn) {
2105 hostParallelCallback(OMPIRBuilder: this, OutlinedFn, OuterFn, Ident, IfCondition,
2106 PrivTID, PrivTIDAddr, ToBeDeleted: ToBeDeletedVec);
2107 };
2108 }
2109
2110 OI->FixUpNonEntryAllocas = true;
2111 OI->OuterAllocBB = OuterAllocaBlock;
2112 OI->EntryBB = PRegEntryBB;
2113 OI->ExitBB = PRegExitBB;
2114 OI->OuterDeallocBBs.reserve(N: OuterDeallocBlocks.size());
2115 copy(Range&: OuterDeallocBlocks, Out: OI->OuterDeallocBBs.end());
2116
2117 SmallPtrSet<BasicBlock *, 32> ParallelRegionBlockSet;
2118 SmallVector<BasicBlock *, 32> Blocks;
2119 OI->collectBlocks(BlockSet&: ParallelRegionBlockSet, BlockVector&: Blocks);
2120
2121 CodeExtractorAnalysisCache CEAC(*OuterFn);
2122 CodeExtractor Extractor(Blocks, /* DominatorTree */ nullptr,
2123 /* AggregateArgs */ false,
2124 /* BlockFrequencyInfo */ nullptr,
2125 /* BranchProbabilityInfo */ nullptr,
2126 /* AssumptionCache */ nullptr,
2127 /* AllowVarArgs */ true,
2128 /* AllowAlloca */ true,
2129 /* AllocationBlock */ OuterAllocaBlock,
2130 /* DeallocationBlocks */ {},
2131 /* Suffix */ ".omp_par", ArgsInZeroAddressSpace);
2132
2133 // Find inputs to, outputs from the code region.
2134 BasicBlock *CommonExit = nullptr;
2135 SetVector<Value *> Inputs, Outputs, SinkingCands, HoistingCands;
2136 Extractor.findAllocas(CEAC, SinkCands&: SinkingCands, HoistCands&: HoistingCands, ExitBlock&: CommonExit);
2137
2138 Extractor.findInputsOutputs(Inputs, Outputs, Allocas: SinkingCands,
2139 /*CollectGlobalInputs=*/true);
2140
2141 Inputs.remove_if(P: [&](Value *I) {
2142 if (auto *GV = dyn_cast_if_present<GlobalVariable>(Val: I))
2143 return GV->getValueType() == OpenMPIRBuilder::Ident;
2144
2145 return false;
2146 });
2147
2148 LLVM_DEBUG(dbgs() << "Before privatization: " << *OuterFn << "\n");
2149
2150 FunctionCallee TIDRTLFn =
2151 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_global_thread_num);
2152
2153 auto PrivHelper = [&](Value &V) -> Error {
2154 if (&V == TIDAddr || &V == ZeroAddr) {
2155 OI->ExcludeArgsFromAggregate.push_back(Elt: &V);
2156 return Error::success();
2157 }
2158
2159 SetVector<Use *> Uses;
2160 for (Use &U : V.uses())
2161 if (auto *UserI = dyn_cast<Instruction>(Val: U.getUser()))
2162 if (ParallelRegionBlockSet.count(Ptr: UserI->getParent()))
2163 Uses.insert(X: &U);
2164
2165 // __kmpc_fork_call expects extra arguments as pointers. If the input
2166 // already has a pointer type, everything is fine. Otherwise, store the
2167 // value onto stack and load it back inside the to-be-outlined region. This
2168 // will ensure only the pointer will be passed to the function.
2169 // FIXME: if there are more than 15 trailing arguments, they must be
2170 // additionally packed in a struct.
2171 Value *Inner = &V;
2172 if (!V.getType()->isPointerTy()) {
2173 IRBuilder<>::InsertPointGuard Guard(Builder);
2174 LLVM_DEBUG(llvm::dbgs() << "Forwarding input as pointer: " << V << "\n");
2175
2176 Builder.restoreIP(IP: OuterAllocIP);
2177 Value *Ptr;
2178 if (UsesDeviceSharedMemory) {
2179 // Use device shared memory instead, if needed.
2180 Ptr = createOMPAllocShared(Loc: OuterAllocIP, VarType: V.getType(),
2181 Name: V.getName() + ".reloaded");
2182 for (BasicBlock *DeallocBlock : OuterDeallocBlocks)
2183 createOMPFreeShared(
2184 Loc: InsertPointTy(DeallocBlock, DeallocBlock->getFirstInsertionPt()),
2185 Addr: Ptr, VarType: V.getType());
2186 } else {
2187 Ptr = Builder.CreateAlloca(Ty: V.getType(), ArraySize: nullptr,
2188 Name: V.getName() + ".reloaded");
2189 }
2190
2191 // Store to stack at end of the block that currently branches to the entry
2192 // block of the to-be-outlined region.
2193 Builder.SetInsertPoint(TheBB: InsertBB,
2194 IP: InsertBB->getTerminator()->getIterator());
2195 Builder.CreateStore(Val: &V, Ptr);
2196
2197 // Load back next to allocations in the to-be-outlined region.
2198 Builder.restoreIP(IP: InnerAllocaIP);
2199 Inner = Builder.CreateLoad(Ty: V.getType(), Ptr);
2200 }
2201
2202 Value *ReplacementValue = nullptr;
2203 CallInst *CI = dyn_cast<CallInst>(Val: &V);
2204 if (CI && CI->getCalledFunction() == TIDRTLFn.getCallee()) {
2205 ReplacementValue = PrivTID;
2206 } else {
2207 InsertPointOrErrorTy AfterIP =
2208 PrivCB(InnerAllocaIP, Builder.saveIP(), V, *Inner, ReplacementValue);
2209 if (!AfterIP)
2210 return AfterIP.takeError();
2211 Builder.restoreIP(IP: *AfterIP);
2212 InnerAllocaIP = {
2213 InnerAllocaIP.getBlock(),
2214 InnerAllocaIP.getBlock()->getTerminator()->getIterator()};
2215
2216 assert(ReplacementValue &&
2217 "Expected copy/create callback to set replacement value!");
2218 if (ReplacementValue == &V)
2219 return Error::success();
2220 }
2221
2222 for (Use *UPtr : Uses)
2223 UPtr->set(ReplacementValue);
2224
2225 return Error::success();
2226 };
2227
2228 // Reset the inner alloca insertion as it will be used for loading the values
2229 // wrapped into pointers before passing them into the to-be-outlined region.
2230 // Configure it to insert immediately after the fake use of zero address so
2231 // that they are available in the generated body and so that the
2232 // OpenMP-related values (thread ID and zero address pointers) remain leading
2233 // in the argument list.
2234 InnerAllocaIP = IRBuilder<>::InsertPoint(
2235 ZeroAddrUse->getParent(), ZeroAddrUse->getNextNode()->getIterator());
2236
2237 // Reset the outer alloca insertion point to the entry of the relevant block
2238 // in case it was invalidated.
2239 OuterAllocIP = IRBuilder<>::InsertPoint(
2240 OuterAllocaBlock, OuterAllocaBlock->getFirstInsertionPt());
2241
2242 for (Value *Input : Inputs) {
2243 LLVM_DEBUG(dbgs() << "Captured input: " << *Input << "\n");
2244 if (Error Err = PrivHelper(*Input))
2245 return Err;
2246 }
2247 LLVM_DEBUG({
2248 for (Value *Output : Outputs)
2249 LLVM_DEBUG(dbgs() << "Captured output: " << *Output << "\n");
2250 });
2251 assert(Outputs.empty() &&
2252 "OpenMP outlining should not produce live-out values!");
2253
2254 LLVM_DEBUG(dbgs() << "After privatization: " << *OuterFn << "\n");
2255 LLVM_DEBUG({
2256 for (auto *BB : Blocks)
2257 dbgs() << " PBR: " << BB->getName() << "\n";
2258 });
2259
2260 // Adjust the finalization stack, verify the adjustment, and call the
2261 // finalize function a last time to finalize values between the pre-fini
2262 // block and the exit block if we left the parallel "the normal way".
2263 auto FiniInfo = FinalizationStack.pop_back_val();
2264 (void)FiniInfo;
2265 assert(FiniInfo.DK == OMPD_parallel &&
2266 "Unexpected finalization stack state!");
2267
2268 Instruction *PRegPreFiniTI = PRegPreFiniBB->getTerminator();
2269
2270 InsertPointTy PreFiniIP(PRegPreFiniBB, PRegPreFiniTI->getIterator());
2271 Expected<BasicBlock *> FiniBBOrErr = FiniInfo.getFiniBB(Builder);
2272 if (!FiniBBOrErr)
2273 return FiniBBOrErr.takeError();
2274 {
2275 IRBuilderBase::InsertPointGuard Guard(Builder);
2276 Builder.restoreIP(IP: PreFiniIP);
2277 Builder.CreateBr(Dest: *FiniBBOrErr);
2278 // There's currently a branch to omp.par.exit. Delete it. We will get there
2279 // via the fini block
2280 if (Instruction *Term = Builder.GetInsertBlock()->getTerminator())
2281 Term->eraseFromParent();
2282 }
2283
2284 // Register the outlined info.
2285 addOutlineInfo(OI: std::move(OI));
2286
2287 InsertPointTy AfterIP(UI->getParent(), UI->getParent()->end());
2288 UI->eraseFromParent();
2289
2290 return AfterIP;
2291}
2292
2293void OpenMPIRBuilder::emitFlush(const LocationDescription &Loc) {
2294 // Build call void __kmpc_flush(ident_t *loc)
2295 uint32_t SrcLocStrSize;
2296 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2297 Value *Args[] = {getOrCreateIdent(SrcLocStr, SrcLocStrSize)};
2298
2299 createRuntimeFunctionCall(Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_flush),
2300 Args);
2301}
2302
2303void OpenMPIRBuilder::createFlush(const LocationDescription &Loc) {
2304 if (!updateToLocation(Loc))
2305 return;
2306 emitFlush(Loc);
2307}
2308
2309void OpenMPIRBuilder::createError(const LocationDescription &Loc, bool IsFatal,
2310 Value *Message) {
2311 if (!updateToLocation(Loc))
2312 return;
2313
2314 // Build call void __kmpc_error(ident_t *loc, int severity,
2315 // const char *message)
2316 uint32_t SrcLocStrSize;
2317 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2318 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2319 // Severity: 1 = warning, 2 = fatal.
2320 Value *Severity = ConstantInt::get(Ty: Int32, V: IsFatal ? 2 : 1);
2321 Value *MessageArg = Message ? Message : ConstantPointerNull::get(T: Int8Ptr);
2322 Value *Args[] = {Ident, Severity, MessageArg};
2323
2324 createRuntimeFunctionCall(Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_error),
2325 Args);
2326}
2327
2328void OpenMPIRBuilder::emitTaskyieldImpl(const LocationDescription &Loc) {
2329 // Build call __kmpc_omp_taskyield(loc, thread_id, 0);
2330 uint32_t SrcLocStrSize;
2331 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2332 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2333 Constant *I32Null = ConstantInt::getNullValue(Ty: Int32);
2334 Value *Args[] = {Ident, getOrCreateThreadID(Ident), I32Null};
2335
2336 createRuntimeFunctionCall(
2337 Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_taskyield), Args);
2338}
2339
2340void OpenMPIRBuilder::createTaskyield(const LocationDescription &Loc) {
2341 if (!updateToLocation(Loc))
2342 return;
2343 emitTaskyieldImpl(Loc);
2344}
2345
2346void OpenMPIRBuilder::emitTaskDependency(IRBuilderBase &Builder, Value *Entry,
2347 const DependData &Dep) {
2348 // Store the pointer to the variable
2349 Value *Addr = Builder.CreateStructGEP(
2350 Ty: DependInfo, Ptr: Entry,
2351 Idx: static_cast<unsigned int>(RTLDependInfoFields::BaseAddr));
2352 Value *DepValPtr = Builder.CreatePtrToInt(V: Dep.DepVal, DestTy: SizeTy);
2353 Builder.CreateStore(Val: DepValPtr, Ptr: Addr);
2354 // Store the size of the variable
2355 Value *Size = Builder.CreateStructGEP(
2356 Ty: DependInfo, Ptr: Entry, Idx: static_cast<unsigned int>(RTLDependInfoFields::Len));
2357 Builder.CreateStore(
2358 Val: ConstantInt::get(Ty: SizeTy,
2359 V: M.getDataLayout().getTypeStoreSize(Ty: Dep.DepValueType)),
2360 Ptr: Size);
2361 // Store the dependency kind
2362 Value *Flags = Builder.CreateStructGEP(
2363 Ty: DependInfo, Ptr: Entry, Idx: static_cast<unsigned int>(RTLDependInfoFields::Flags));
2364 Builder.CreateStore(Val: ConstantInt::get(Ty: Builder.getInt8Ty(),
2365 V: static_cast<unsigned int>(Dep.DepKind)),
2366 Ptr: Flags);
2367}
2368
2369// Processes the dependencies in Dependencies and does the following
2370// - Allocates space on the stack of an array of DependInfo objects
2371// - Populates each DependInfo object with relevant information of
2372// the corresponding dependence.
2373// - All code is inserted in the entry block of the current function.
2374static Value *emitTaskDependencies(
2375 OpenMPIRBuilder &OMPBuilder,
2376 const SmallVectorImpl<OpenMPIRBuilder::DependData> &Dependencies) {
2377 // Early return if we have no dependencies to process
2378 if (Dependencies.empty())
2379 return nullptr;
2380
2381 // Given a vector of DependData objects, in this function we create an
2382 // array on the stack that holds kmp_depend_info objects corresponding
2383 // to each dependency. This is then passed to the OpenMP runtime.
2384 // For example, if there are 'n' dependencies then the following psedo
2385 // code is generated. Assume the first dependence is on a variable 'a'
2386 //
2387 // \code{c}
2388 // DepArray = alloc(n x sizeof(kmp_depend_info);
2389 // idx = 0;
2390 // DepArray[idx].base_addr = ptrtoint(&a);
2391 // DepArray[idx].len = 8;
2392 // DepArray[idx].flags = Dep.DepKind; /*(See OMPContants.h for DepKind)*/
2393 // ++idx;
2394 // DepArray[idx].base_addr = ...;
2395 // \endcode
2396
2397 IRBuilderBase &Builder = OMPBuilder.Builder;
2398 Type *DependInfo = OMPBuilder.DependInfo;
2399
2400 Value *DepArray = nullptr;
2401 OpenMPIRBuilder::InsertPointTy OldIP = Builder.saveIP();
2402 Builder.SetInsertPoint(
2403 OldIP.getBlock()->getParent()->getEntryBlock().getTerminator());
2404
2405 Type *DepArrayTy = ArrayType::get(ElementType: DependInfo, NumElements: Dependencies.size());
2406 DepArray = Builder.CreateAlloca(Ty: DepArrayTy, ArraySize: nullptr, Name: ".dep.arr.addr");
2407
2408 Builder.restoreIP(IP: OldIP);
2409
2410 for (const auto &[DepIdx, Dep] : enumerate(First: Dependencies)) {
2411 Value *Base =
2412 Builder.CreateConstInBoundsGEP2_64(Ty: DepArrayTy, Ptr: DepArray, Idx0: 0, Idx1: DepIdx);
2413 OMPBuilder.emitTaskDependency(Builder, Entry: Base, Dep);
2414 }
2415 return DepArray;
2416}
2417
2418void OpenMPIRBuilder::emitTaskwaitImpl(const LocationDescription &Loc) {
2419 // Build call kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
2420 // global_tid);
2421 uint32_t SrcLocStrSize;
2422 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2423 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2424 Value *Args[] = {Ident, getOrCreateThreadID(Ident)};
2425
2426 // Ignore return result until untied tasks are supported.
2427 createRuntimeFunctionCall(
2428 Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_taskwait), Args);
2429}
2430
2431void OpenMPIRBuilder::createTaskwait(const LocationDescription &Loc,
2432 DependenciesInfo Dependencies) {
2433 if (!updateToLocation(Loc))
2434 return;
2435
2436 Value *DepArray = nullptr;
2437 Type *DepArrayTy = nullptr;
2438 Value *NumDeps = nullptr;
2439 if (Dependencies.DepArray) {
2440 DepArray = Dependencies.DepArray;
2441 NumDeps = Dependencies.NumDeps;
2442 } else if (!Dependencies.Deps.empty()) {
2443 InsertPointTy OldIP = Builder.saveIP();
2444 BasicBlock &entryBB =
2445 Builder.GetInsertBlock()->getParent()->getEntryBlock();
2446 Builder.SetInsertPoint(TheBB: &entryBB, IP: entryBB.getFirstInsertionPt());
2447
2448 DepArrayTy = ArrayType::get(ElementType: DependInfo, NumElements: Dependencies.Deps.size());
2449 DepArray = Builder.CreateAlloca(Ty: DepArrayTy, ArraySize: nullptr, Name: ".dep.arr.addr");
2450 NumDeps = Builder.getInt32(C: Dependencies.Deps.size());
2451
2452 Builder.restoreIP(IP: OldIP);
2453 for (const auto &[DepIdx, Dep] : enumerate(First&: Dependencies.Deps)) {
2454 Value *Base =
2455 Builder.CreateConstInBoundsGEP2_64(Ty: DepArrayTy, Ptr: DepArray, Idx0: 0, Idx1: DepIdx);
2456 this->emitTaskDependency(Builder, Entry: Base, Dep);
2457 }
2458 }
2459
2460 if (DepArray) {
2461 uint32_t SrcLocStrSize;
2462 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2463 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2464 Value *Args[] = {
2465 Ident,
2466 getOrCreateThreadID(Ident),
2467 NumDeps,
2468 DepArray,
2469 ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0),
2470 ConstantPointerNull::get(T: PointerType::getUnqual(C&: M.getContext())),
2471 ConstantInt::get(Ty: Builder.getInt32Ty(), V: false)};
2472 createRuntimeFunctionCall(
2473 Callee: getOrCreateRuntimeFunctionPtr(
2474 FnID: omp::RuntimeFunction::OMPRTL___kmpc_omp_taskwait_deps_51),
2475 Args);
2476 } else {
2477 emitTaskwaitImpl(Loc);
2478 }
2479}
2480
2481/// Create the task duplication function passed to kmpc_taskloop.
2482Expected<Value *> OpenMPIRBuilder::createTaskDuplicationFunction(
2483 Type *PrivatesTy, int32_t PrivatesIndex, TaskDupCallbackTy DupCB) {
2484 unsigned ProgramAddressSpace = M.getDataLayout().getProgramAddressSpace();
2485 if (!DupCB)
2486 return Constant::getNullValue(
2487 Ty: PointerType::get(C&: Builder.getContext(), AddressSpace: ProgramAddressSpace));
2488
2489 // From OpenMP Runtime p_task_dup_t:
2490 // Routine optionally generated by the compiler for setting the lastprivate
2491 // flag and calling needed constructors for private/firstprivate objects (used
2492 // to form taskloop tasks from pattern task) Parameters: dest task, src task,
2493 // lastprivate flag.
2494 // typedef void (*p_task_dup_t)(kmp_task_t *, kmp_task_t *, kmp_int32);
2495
2496 auto *VoidPtrTy = PointerType::get(C&: Builder.getContext(), AddressSpace: ProgramAddressSpace);
2497
2498 FunctionType *DupFuncTy = FunctionType::get(
2499 Result: Builder.getVoidTy(), Params: {VoidPtrTy, VoidPtrTy, Builder.getInt32Ty()},
2500 /*isVarArg=*/false);
2501
2502 Function *DupFunction = Function::Create(Ty: DupFuncTy, Linkage: Function::InternalLinkage,
2503 N: "omp_taskloop_dup", M);
2504 Value *DestTaskArg = DupFunction->getArg(i: 0);
2505 Value *SrcTaskArg = DupFunction->getArg(i: 1);
2506 Value *LastprivateFlagArg = DupFunction->getArg(i: 2);
2507 DestTaskArg->setName("dest_task");
2508 SrcTaskArg->setName("src_task");
2509 LastprivateFlagArg->setName("lastprivate_flag");
2510
2511 IRBuilderBase::InsertPointGuard Guard(Builder);
2512 Builder.SetInsertPoint(
2513 BasicBlock::Create(Context&: Builder.getContext(), Name: "entry", Parent: DupFunction));
2514
2515 auto GetTaskContextPtrFromArg = [&](Value *Arg) -> Value * {
2516 Type *TaskWithPrivatesTy =
2517 StructType::get(Context&: Builder.getContext(), Elements: {Task, PrivatesTy});
2518 Value *TaskPrivates = Builder.CreateGEP(
2519 Ty: TaskWithPrivatesTy, Ptr: Arg, IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: 1)});
2520 Value *ContextPtr = Builder.CreateGEP(
2521 Ty: PrivatesTy, Ptr: TaskPrivates,
2522 IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: PrivatesIndex)});
2523 return ContextPtr;
2524 };
2525
2526 Value *DestTaskContextPtr = GetTaskContextPtrFromArg(DestTaskArg);
2527 Value *SrcTaskContextPtr = GetTaskContextPtrFromArg(SrcTaskArg);
2528
2529 DestTaskContextPtr->setName("destPtr");
2530 SrcTaskContextPtr->setName("srcPtr");
2531
2532 InsertPointTy AllocaIP(&DupFunction->getEntryBlock(),
2533 DupFunction->getEntryBlock().begin());
2534 InsertPointTy CodeGenIP = Builder.saveIP();
2535 Expected<IRBuilderBase::InsertPoint> AfterIPOrError =
2536 DupCB(AllocaIP, CodeGenIP, DestTaskContextPtr, SrcTaskContextPtr);
2537 if (!AfterIPOrError)
2538 return AfterIPOrError.takeError();
2539 Builder.restoreIP(IP: *AfterIPOrError);
2540
2541 Builder.CreateRetVoid();
2542
2543 return DupFunction;
2544}
2545
2546OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTaskloop(
2547 const LocationDescription &Loc, InsertPointTy AllocaIP,
2548 ArrayRef<BasicBlock *> DeallocBlocks, BodyGenCallbackTy BodyGenCB,
2549 llvm::function_ref<llvm::Expected<llvm::CanonicalLoopInfo *>()> LoopInfo,
2550 Value *LBVal, Value *UBVal, Value *StepVal, bool Untied, Value *IfCond,
2551 Value *GrainSize, bool NoGroup, int Sched, Value *Final, bool Mergeable,
2552 Value *Priority, uint64_t NumOfCollapseLoops, TaskDupCallbackTy DupCB,
2553 Value *TaskContextStructPtrVal) {
2554
2555 if (!updateToLocation(Loc))
2556 return InsertPointTy();
2557
2558 uint32_t SrcLocStrSize;
2559 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2560 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2561
2562 BasicBlock *TaskloopExitBB =
2563 splitBB(Builder, /*CreateBranch=*/true, Name: "taskloop.exit");
2564 BasicBlock *TaskloopBodyBB =
2565 splitBB(Builder, /*CreateBranch=*/true, Name: "taskloop.body");
2566 BasicBlock *TaskloopAllocaBB =
2567 splitBB(Builder, /*CreateBranch=*/true, Name: "taskloop.alloca");
2568
2569 InsertPointTy TaskloopAllocaIP =
2570 InsertPointTy(TaskloopAllocaBB, TaskloopAllocaBB->begin());
2571 InsertPointTy TaskloopBodyIP =
2572 InsertPointTy(TaskloopBodyBB, TaskloopBodyBB->begin());
2573
2574 if (Error Err = BodyGenCB(TaskloopAllocaIP, TaskloopBodyIP, TaskloopExitBB))
2575 return Err;
2576
2577 llvm::Expected<llvm::CanonicalLoopInfo *> result = LoopInfo();
2578 if (!result) {
2579 return result.takeError();
2580 }
2581
2582 llvm::CanonicalLoopInfo *CLI = result.get();
2583 auto OI = std::make_unique<OutlineInfo>();
2584 OI->EntryBB = TaskloopAllocaBB;
2585 OI->OuterAllocBB = AllocaIP.getBlock();
2586 OI->ExitBB = TaskloopExitBB;
2587 OI->OuterDeallocBBs.reserve(N: DeallocBlocks.size());
2588 copy(Range&: DeallocBlocks, Out: OI->OuterDeallocBBs.end());
2589
2590 // Add the thread ID argument.
2591 SmallVector<Instruction *> ToBeDeleted;
2592 // dummy instruction to be used as a fake argument
2593 OI->ExcludeArgsFromAggregate.push_back(Elt: createFakeIntVal(
2594 Builder, OuterAllocaIP: AllocaIP, ToBeDeleted, InnerAllocaIP: TaskloopAllocaIP, Name: "global.tid", AsPtr: false));
2595 Value *FakeLB = createFakeIntVal(Builder, OuterAllocaIP: AllocaIP, ToBeDeleted,
2596 InnerAllocaIP: TaskloopAllocaIP, Name: "lb", AsPtr: false, Is64Bit: true);
2597 Value *FakeUB = createFakeIntVal(Builder, OuterAllocaIP: AllocaIP, ToBeDeleted,
2598 InnerAllocaIP: TaskloopAllocaIP, Name: "ub", AsPtr: false, Is64Bit: true);
2599 Value *FakeStep = createFakeIntVal(Builder, OuterAllocaIP: AllocaIP, ToBeDeleted,
2600 InnerAllocaIP: TaskloopAllocaIP, Name: "step", AsPtr: false, Is64Bit: true);
2601 // For Taskloop, we want to force the bounds being the first 3 inputs in the
2602 // aggregate struct
2603 OI->Inputs.insert(X: FakeLB);
2604 OI->Inputs.insert(X: FakeUB);
2605 OI->Inputs.insert(X: FakeStep);
2606 if (TaskContextStructPtrVal)
2607 OI->Inputs.insert(X: TaskContextStructPtrVal);
2608 assert(((TaskContextStructPtrVal && DupCB) ||
2609 (!TaskContextStructPtrVal && !DupCB)) &&
2610 "Task context struct ptr and duplication callback must be both set "
2611 "or both null");
2612
2613 // It isn't safe to run the duplication bodygen callback inside the post
2614 // outlining callback so this has to be run now before we know the real task
2615 // shareds structure type.
2616 unsigned ProgramAddressSpace = M.getDataLayout().getProgramAddressSpace();
2617 Type *PointerTy = PointerType::get(C&: Builder.getContext(), AddressSpace: ProgramAddressSpace);
2618 Type *FakeSharedsTy = StructType::get(
2619 Context&: Builder.getContext(),
2620 Elements: {FakeLB->getType(), FakeUB->getType(), FakeStep->getType(), PointerTy});
2621 Expected<Value *> TaskDupFnOrErr = createTaskDuplicationFunction(
2622 PrivatesTy: FakeSharedsTy,
2623 /*PrivatesIndex: the pointer after the three indices above*/ PrivatesIndex: 3, DupCB);
2624 if (!TaskDupFnOrErr) {
2625 return TaskDupFnOrErr.takeError();
2626 }
2627 Value *TaskDupFn = *TaskDupFnOrErr;
2628
2629 OI->PostOutlineCB = [this, Ident, LBVal, UBVal, StepVal, Untied,
2630 TaskloopAllocaBB, CLI, TaskDupFn, ToBeDeleted, IfCond,
2631 GrainSize, NoGroup, Sched, FakeLB, FakeUB, FakeStep,
2632 FakeSharedsTy, Final, Mergeable, Priority,
2633 NumOfCollapseLoops](Function &OutlinedFn) mutable {
2634 // Replace the Stale CI by appropriate RTL function call.
2635 assert(OutlinedFn.hasOneUse() &&
2636 "there must be a single user for the outlined function");
2637 CallInst *StaleCI = cast<CallInst>(Val: OutlinedFn.user_back());
2638
2639 /* Create the casting for the Bounds Values that can be used when outlining
2640 * to replace the uses of the fakes with real values */
2641 BasicBlock *CodeReplBB = StaleCI->getParent();
2642 Builder.SetInsertPoint(CodeReplBB->getFirstInsertionPt());
2643 Value *CastedLBVal =
2644 Builder.CreateIntCast(V: LBVal, DestTy: Builder.getInt64Ty(), isSigned: true, Name: "lb64");
2645 Value *CastedUBVal =
2646 Builder.CreateIntCast(V: UBVal, DestTy: Builder.getInt64Ty(), isSigned: true, Name: "ub64");
2647 Value *CastedStepVal =
2648 Builder.CreateIntCast(V: StepVal, DestTy: Builder.getInt64Ty(), isSigned: true, Name: "step64");
2649
2650 Builder.SetInsertPoint(StaleCI);
2651
2652 // Gather the arguments for emitting the runtime call for
2653 // @__kmpc_omp_task_alloc
2654 Function *TaskAllocFn =
2655 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_alloc);
2656
2657 Value *ThreadID = getOrCreateThreadID(Ident);
2658
2659 if (!NoGroup) {
2660 // Emit runtime call for @__kmpc_taskgroup
2661 Function *TaskgroupFn =
2662 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_taskgroup);
2663 Builder.CreateCall(Callee: TaskgroupFn, Args: {Ident, ThreadID});
2664 }
2665
2666 // `flags` Argument Configuration
2667 // Task is tied if (Flags & 1) == 1.
2668 // Task is untied if (Flags & 1) == 0.
2669 // Task is final if (Flags & 2) == 2.
2670 // Task is not final if (Flags & 2) == 0.
2671 // Task is mergeable if (Flags & 4) == 4.
2672 // Task is not mergeable if (Flags & 4) == 0.
2673 // Task is priority if (Flags & 32) == 32.
2674 // Task is not priority if (Flags & 32) == 0.
2675 Value *Flags = Builder.getInt32(C: Untied ? 0 : 1);
2676 if (Final)
2677 Flags = Builder.CreateOr(LHS: Builder.getInt32(C: 2), RHS: Flags);
2678 if (Mergeable)
2679 Flags = Builder.CreateOr(LHS: Builder.getInt32(C: 4), RHS: Flags);
2680 if (Priority)
2681 Flags = Builder.CreateOr(LHS: Builder.getInt32(C: 32), RHS: Flags);
2682
2683 Value *TaskSize = Builder.getInt64(
2684 C: divideCeil(Numerator: M.getDataLayout().getTypeSizeInBits(Ty: Task), Denominator: 8));
2685
2686 AllocaInst *ArgStructAlloca =
2687 dyn_cast<AllocaInst>(Val: StaleCI->getArgOperand(i: 1));
2688 assert(ArgStructAlloca &&
2689 "Unable to find the alloca instruction corresponding to arguments "
2690 "for extracted function");
2691 std::optional<TypeSize> ArgAllocSize =
2692 ArgStructAlloca->getAllocationSize(DL: M.getDataLayout());
2693 assert(ArgAllocSize &&
2694 "Unable to determine size of arguments for extracted function");
2695 Value *SharedsSize = Builder.getInt64(C: ArgAllocSize->getFixedValue());
2696
2697 // Emit the @__kmpc_omp_task_alloc runtime call
2698 // The runtime call returns a pointer to an area where the task captured
2699 // variables must be copied before the task is run (TaskData)
2700 CallInst *TaskData = Builder.CreateCall(
2701 Callee: TaskAllocFn, Args: {/*loc_ref=*/Ident, /*gtid=*/ThreadID, /*flags=*/Flags,
2702 /*sizeof_task=*/TaskSize, /*sizeof_shared=*/SharedsSize,
2703 /*task_func=*/&OutlinedFn});
2704
2705 Value *Shareds = StaleCI->getArgOperand(i: 1);
2706 Align Alignment = TaskData->getPointerAlignment(DL: M.getDataLayout());
2707 Value *TaskShareds = Builder.CreateLoad(Ty: VoidPtr, Ptr: TaskData);
2708 Builder.CreateMemCpy(Dst: TaskShareds, DstAlign: Alignment, Src: Shareds, SrcAlign: Alignment,
2709 Size: SharedsSize);
2710 // Get the pointer to loop lb, ub, step from task ptr
2711 // and set up the lowerbound,upperbound and step values
2712 llvm::Value *Lb = Builder.CreateGEP(
2713 Ty: FakeSharedsTy, Ptr: TaskShareds, IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: 0)});
2714
2715 llvm::Value *Ub = Builder.CreateGEP(
2716 Ty: FakeSharedsTy, Ptr: TaskShareds, IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: 1)});
2717
2718 llvm::Value *Step = Builder.CreateGEP(
2719 Ty: FakeSharedsTy, Ptr: TaskShareds, IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: 2)});
2720 llvm::Value *Loadstep = Builder.CreateLoad(Ty: Builder.getInt64Ty(), Ptr: Step);
2721
2722 // set up the arguments for emitting kmpc_taskloop runtime call
2723 // setting values for ifval, nogroup, sched, grainsize, task_dup
2724 Value *IfCondVal =
2725 IfCond ? Builder.CreateIntCast(V: IfCond, DestTy: Builder.getInt32Ty(), isSigned: true)
2726 : Builder.getInt32(C: 1);
2727 // As __kmpc_taskgroup is called manually in OMPIRBuilder, NoGroupVal should
2728 // always be 1 when calling __kmpc_taskloop to ensure it is not called again
2729 Value *NoGroupVal = Builder.getInt32(C: 1);
2730 Value *SchedVal = Builder.getInt32(C: Sched);
2731 Value *GrainSizeVal =
2732 GrainSize ? Builder.CreateIntCast(V: GrainSize, DestTy: Builder.getInt64Ty(), isSigned: true)
2733 : Builder.getInt64(C: 0);
2734 Value *TaskDup = TaskDupFn;
2735
2736 Value *Args[] = {Ident, ThreadID, TaskData, IfCondVal, Lb, Ub,
2737 Loadstep, NoGroupVal, SchedVal, GrainSizeVal, TaskDup};
2738
2739 // taskloop runtime call
2740 Function *TaskloopFn =
2741 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_taskloop);
2742 Builder.CreateCall(Callee: TaskloopFn, Args);
2743
2744 // Emit the @__kmpc_end_taskgroup runtime call to end the taskgroup if
2745 // nogroup is not defined
2746 if (!NoGroup) {
2747 Function *EndTaskgroupFn =
2748 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_end_taskgroup);
2749 Builder.CreateCall(Callee: EndTaskgroupFn, Args: {Ident, ThreadID});
2750 }
2751
2752 StaleCI->eraseFromParent();
2753
2754 Builder.SetInsertPoint(TheBB: TaskloopAllocaBB, IP: TaskloopAllocaBB->begin());
2755
2756 LoadInst *SharedsOutlined =
2757 Builder.CreateLoad(Ty: VoidPtr, Ptr: OutlinedFn.getArg(i: 1));
2758 OutlinedFn.getArg(i: 1)->replaceUsesWithIf(
2759 New: SharedsOutlined,
2760 ShouldReplace: [SharedsOutlined](Use &U) { return U.getUser() != SharedsOutlined; });
2761
2762 Value *IV = CLI->getIndVar();
2763 Type *IVTy = IV->getType();
2764 Constant *One = ConstantInt::get(Ty: Builder.getInt64Ty(), V: 1);
2765
2766 // When outlining, CodeExtractor will create GEP's to the LowerBound and
2767 // UpperBound. These GEP's can be reused for loading the tasks respective
2768 // bounds.
2769 Value *TaskLB = nullptr;
2770 Value *TaskUB = nullptr;
2771 Value *TaskStep = nullptr;
2772 Value *LoadTaskLB = nullptr;
2773 Value *LoadTaskUB = nullptr;
2774 Value *LoadTaskStep = nullptr;
2775 for (Instruction &I : *TaskloopAllocaBB) {
2776 if (I.getOpcode() == Instruction::GetElementPtr) {
2777 GetElementPtrInst &Gep = cast<GetElementPtrInst>(Val&: I);
2778 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: Gep.getOperand(i_nocapture: 2))) {
2779 switch (CI->getZExtValue()) {
2780 case 0:
2781 TaskLB = &I;
2782 break;
2783 case 1:
2784 TaskUB = &I;
2785 break;
2786 case 2:
2787 TaskStep = &I;
2788 break;
2789 }
2790 }
2791 } else if (I.getOpcode() == Instruction::Load) {
2792 LoadInst &Load = cast<LoadInst>(Val&: I);
2793 if (Load.getPointerOperand() == TaskLB) {
2794 assert(TaskLB != nullptr && "Expected value for TaskLB");
2795 LoadTaskLB = &I;
2796 } else if (Load.getPointerOperand() == TaskUB) {
2797 assert(TaskUB != nullptr && "Expected value for TaskUB");
2798 LoadTaskUB = &I;
2799 } else if (Load.getPointerOperand() == TaskStep) {
2800 assert(TaskStep != nullptr && "Expected value for TaskStep");
2801 LoadTaskStep = &I;
2802 }
2803 }
2804 }
2805
2806 Builder.SetInsertPoint(CLI->getPreheader()->getTerminator());
2807
2808 assert(LoadTaskLB != nullptr && "Expected value for LoadTaskLB");
2809 assert(LoadTaskUB != nullptr && "Expected value for LoadTaskUB");
2810 assert(LoadTaskStep != nullptr && "Expected value for LoadTaskStep");
2811 Value *TripCountMinusOne = Builder.CreateSDiv(
2812 LHS: Builder.CreateSub(LHS: LoadTaskUB, RHS: LoadTaskLB), RHS: LoadTaskStep);
2813 Value *TripCount = Builder.CreateAdd(LHS: TripCountMinusOne, RHS: One, Name: "trip_cnt");
2814 Value *CastedTripCount = Builder.CreateIntCast(V: TripCount, DestTy: IVTy, isSigned: true);
2815 Value *CastedTaskLB = Builder.CreateIntCast(V: LoadTaskLB, DestTy: IVTy, isSigned: true);
2816 // set the trip count in the CLI
2817 CLI->setTripCount(CastedTripCount);
2818
2819 Builder.SetInsertPoint(TheBB: CLI->getBody(),
2820 IP: CLI->getBody()->getFirstInsertionPt());
2821
2822 if (NumOfCollapseLoops > 1) {
2823 llvm::SmallVector<User *> UsersToReplace;
2824 // When using the collapse clause, the bounds of the loop have to be
2825 // adjusted to properly represent the iterator of the outer loop.
2826 Value *IVPlusTaskLB = Builder.CreateAdd(
2827 LHS: CLI->getIndVar(),
2828 RHS: Builder.CreateSub(LHS: CastedTaskLB, RHS: ConstantInt::get(Ty: IVTy, V: 1)));
2829 // To ensure every Use is correctly captured, we first want to record
2830 // which users to replace the value in, and then replace the value.
2831 for (auto IVUse = CLI->getIndVar()->uses().begin();
2832 IVUse != CLI->getIndVar()->uses().end(); IVUse++) {
2833 User *IVUser = IVUse->getUser();
2834 if (auto *Op = dyn_cast<BinaryOperator>(Val: IVUser)) {
2835 if (Op->getOpcode() == Instruction::URem ||
2836 Op->getOpcode() == Instruction::UDiv) {
2837 UsersToReplace.push_back(Elt: IVUser);
2838 }
2839 }
2840 }
2841 for (User *User : UsersToReplace) {
2842 User->replaceUsesOfWith(From: CLI->getIndVar(), To: IVPlusTaskLB);
2843 }
2844 } else {
2845 // The canonical loop is generated with a fixed lower bound. We need to
2846 // update the index calculation code to use the task's lower bound. The
2847 // generated code looks like this:
2848 // %omp_loop.iv = phi ...
2849 // ...
2850 // %tmp = mul [type] %omp_loop.iv, step
2851 // %user_index = add [type] tmp, lb
2852 // OpenMPIRBuilder constructs canonical loops to have exactly three uses
2853 // of the normalised induction variable:
2854 // 1. This one: converting the normalised IV to the user IV
2855 // 2. The increment (add)
2856 // 3. The comparison against the trip count (icmp)
2857 // (1) is the only use that is a mul followed by an add so this cannot
2858 // match other IR.
2859 assert(CLI->getIndVar()->getNumUses() == 3 &&
2860 "Canonical loop should have exactly three uses of the ind var");
2861 for (User *IVUser : CLI->getIndVar()->users()) {
2862 if (auto *Mul = dyn_cast<BinaryOperator>(Val: IVUser)) {
2863 if (Mul->getOpcode() == Instruction::Mul) {
2864 for (User *MulUser : Mul->users()) {
2865 if (auto *Add = dyn_cast<BinaryOperator>(Val: MulUser)) {
2866 if (Add->getOpcode() == Instruction::Add) {
2867 Add->setOperand(i_nocapture: 1, Val_nocapture: CastedTaskLB);
2868 }
2869 }
2870 }
2871 }
2872 }
2873 }
2874 }
2875
2876 FakeLB->replaceAllUsesWith(V: CastedLBVal);
2877 FakeUB->replaceAllUsesWith(V: CastedUBVal);
2878 FakeStep->replaceAllUsesWith(V: CastedStepVal);
2879 for (Instruction *I : llvm::reverse(C&: ToBeDeleted)) {
2880 I->eraseFromParent();
2881 }
2882 };
2883
2884 addOutlineInfo(OI: std::move(OI));
2885 Builder.SetInsertPoint(TheBB: TaskloopExitBB, IP: TaskloopExitBB->begin());
2886 return Builder.saveIP();
2887}
2888
2889llvm::StructType *OpenMPIRBuilder::getKmpTaskAffinityInfoTy() {
2890 llvm::Type *IntPtrTy = llvm::Type::getIntNTy(
2891 C&: M.getContext(), N: M.getDataLayout().getPointerSizeInBits());
2892 return llvm::StructType::get(elt1: IntPtrTy, elts: IntPtrTy,
2893 elts: llvm::Type::getInt32Ty(C&: M.getContext()));
2894}
2895
2896OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTask(
2897 const LocationDescription &Loc, InsertPointTy AllocaIP,
2898 ArrayRef<BasicBlock *> DeallocBlocks, BodyGenCallbackTy BodyGenCB,
2899 bool Tied, Value *Final, Value *IfCondition,
2900 const DependenciesInfo &Dependencies, const AffinityData &Affinities,
2901 bool Mergeable, Value *EventHandle, Value *Priority) {
2902
2903 if (!updateToLocation(Loc))
2904 return InsertPointTy();
2905
2906 uint32_t SrcLocStrSize;
2907 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2908 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2909 // The current basic block is split into four basic blocks. After outlining,
2910 // they will be mapped as follows:
2911 // ```
2912 // def current_fn() {
2913 // current_basic_block:
2914 // br label %task.exit
2915 // task.exit:
2916 // ; instructions after task
2917 // }
2918 // def outlined_fn() {
2919 // task.alloca:
2920 // br label %task.body
2921 // task.body:
2922 // ret void
2923 // }
2924 // ```
2925 BasicBlock *TaskExitBB = splitBB(Builder, /*CreateBranch=*/true, Name: "task.exit");
2926 BasicBlock *TaskBodyBB = splitBB(Builder, /*CreateBranch=*/true, Name: "task.body");
2927 BasicBlock *TaskAllocaBB =
2928 splitBB(Builder, /*CreateBranch=*/true, Name: "task.alloca");
2929
2930 InsertPointTy TaskAllocaIP =
2931 InsertPointTy(TaskAllocaBB, TaskAllocaBB->begin());
2932 InsertPointTy TaskBodyIP = InsertPointTy(TaskBodyBB, TaskBodyBB->begin());
2933 if (Error Err = BodyGenCB(TaskAllocaIP, TaskBodyIP, TaskExitBB))
2934 return Err;
2935
2936 auto OI = std::make_unique<OutlineInfo>();
2937 OI->EntryBB = TaskAllocaBB;
2938 OI->OuterAllocBB = AllocaIP.getBlock();
2939 OI->ExitBB = TaskExitBB;
2940 OI->OuterDeallocBBs.reserve(N: DeallocBlocks.size());
2941 copy(Range&: DeallocBlocks, Out: OI->OuterDeallocBBs.end());
2942
2943 // Add the thread ID argument.
2944 SmallVector<Instruction *, 4> ToBeDeleted;
2945 OI->ExcludeArgsFromAggregate.push_back(Elt: createFakeIntVal(
2946 Builder, OuterAllocaIP: AllocaIP, ToBeDeleted, InnerAllocaIP: TaskAllocaIP, Name: "global.tid", AsPtr: false));
2947
2948 OI->PostOutlineCB = [this, Ident, Tied, Final, IfCondition, Dependencies,
2949 Affinities, Mergeable, Priority, EventHandle,
2950 TaskAllocaBB,
2951 ToBeDeleted](Function &OutlinedFn) mutable {
2952 // Replace the Stale CI by appropriate RTL function call.
2953 assert(OutlinedFn.hasOneUse() &&
2954 "there must be a single user for the outlined function");
2955 CallInst *StaleCI = cast<CallInst>(Val: OutlinedFn.user_back());
2956
2957 // HasShareds is true if any variables are captured in the outlined region,
2958 // false otherwise.
2959 bool HasShareds = StaleCI->arg_size() > 1;
2960 Builder.SetInsertPoint(StaleCI);
2961
2962 // Gather the arguments for emitting the runtime call for
2963 // @__kmpc_omp_task_alloc
2964 Function *TaskAllocFn =
2965 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_alloc);
2966
2967 // Arguments - `loc_ref` (Ident) and `gtid` (ThreadID)
2968 // call.
2969 Value *ThreadID = getOrCreateThreadID(Ident);
2970
2971 // Argument - `flags`
2972 // Task is tied iff (Flags & 1) == 1.
2973 // Task is untied iff (Flags & 1) == 0.
2974 // Task is final iff (Flags & 2) == 2.
2975 // Task is not final iff (Flags & 2) == 0.
2976 // Task is mergeable or merged-if0 iff (Flags & 4) == 4.
2977 // Task is neither mergeable nor merged-if0 iff (Flags & 4) == 0.
2978 // Task is detachable iff (Flags & 64) == 64.
2979 // Task is not detachable iff (Flags & 64) == 0.
2980 // Task is priority iff (Flags & 32) == 32.
2981 // Task is not priority iff (Flags & 32) == 0.
2982 // TODO: Handle the other flags.
2983 Value *Flags = Builder.getInt32(C: Tied);
2984 auto *ConstIfCondition = dyn_cast_or_null<ConstantInt>(Val: IfCondition);
2985 bool UseMergedIf0Path = ConstIfCondition && ConstIfCondition->isZero();
2986 if (Final) {
2987 Value *FinalFlag =
2988 Builder.CreateSelect(C: Final, True: Builder.getInt32(C: 2), False: Builder.getInt32(C: 0));
2989 Flags = Builder.CreateOr(LHS: FinalFlag, RHS: Flags);
2990 }
2991
2992 if (Mergeable || UseMergedIf0Path)
2993 Flags = Builder.CreateOr(LHS: Builder.getInt32(C: 4), RHS: Flags);
2994 if (EventHandle)
2995 Flags = Builder.CreateOr(LHS: Builder.getInt32(C: 64), RHS: Flags);
2996 if (Priority)
2997 Flags = Builder.CreateOr(LHS: Builder.getInt32(C: 32), RHS: Flags);
2998
2999 // Argument - `sizeof_kmp_task_t` (TaskSize)
3000 // Tasksize refers to the size in bytes of kmp_task_t data structure
3001 // including private vars accessed in task.
3002 // TODO: add kmp_task_t_with_privates (privates)
3003 Value *TaskSize = Builder.getInt64(
3004 C: divideCeil(Numerator: M.getDataLayout().getTypeSizeInBits(Ty: Task), Denominator: 8));
3005
3006 // Argument - `sizeof_shareds` (SharedsSize)
3007 // SharedsSize refers to the shareds array size in the kmp_task_t data
3008 // structure.
3009 Value *SharedsSize = Builder.getInt64(C: 0);
3010 if (HasShareds) {
3011 AllocaInst *ArgStructAlloca =
3012 dyn_cast<AllocaInst>(Val: StaleCI->getArgOperand(i: 1));
3013 assert(ArgStructAlloca &&
3014 "Unable to find the alloca instruction corresponding to arguments "
3015 "for extracted function");
3016 std::optional<TypeSize> ArgAllocSize =
3017 ArgStructAlloca->getAllocationSize(DL: M.getDataLayout());
3018 assert(ArgAllocSize &&
3019 "Unable to determine size of arguments for extracted function");
3020 SharedsSize = Builder.getInt64(C: ArgAllocSize->getFixedValue());
3021 }
3022 // Emit the @__kmpc_omp_task_alloc runtime call
3023 // The runtime call returns a pointer to an area where the task captured
3024 // variables must be copied before the task is run (TaskData)
3025 CallInst *TaskData = createRuntimeFunctionCall(
3026 Callee: TaskAllocFn, Args: {/*loc_ref=*/Ident, /*gtid=*/ThreadID, /*flags=*/Flags,
3027 /*sizeof_task=*/TaskSize, /*sizeof_shared=*/SharedsSize,
3028 /*task_func=*/&OutlinedFn});
3029
3030 if (Affinities.Count && Affinities.Info) {
3031 Function *RegAffFn = getOrCreateRuntimeFunctionPtr(
3032 FnID: OMPRTL___kmpc_omp_reg_task_with_affinity);
3033
3034 createRuntimeFunctionCall(Callee: RegAffFn, Args: {Ident, ThreadID, TaskData,
3035 Affinities.Count, Affinities.Info});
3036 }
3037
3038 // Emit detach clause initialization.
3039 // evt = (typeof(evt))__kmpc_task_allow_completion_event(loc, tid,
3040 // task_descriptor);
3041 if (EventHandle) {
3042 Function *TaskDetachFn = getOrCreateRuntimeFunctionPtr(
3043 FnID: OMPRTL___kmpc_task_allow_completion_event);
3044 llvm::Value *EventVal =
3045 createRuntimeFunctionCall(Callee: TaskDetachFn, Args: {Ident, ThreadID, TaskData});
3046 llvm::Value *EventHandleAddr =
3047 Builder.CreatePointerBitCastOrAddrSpaceCast(V: EventHandle,
3048 DestTy: Builder.getPtrTy(AddrSpace: 0));
3049 EventVal = Builder.CreatePtrToInt(V: EventVal, DestTy: Builder.getInt64Ty());
3050 Builder.CreateStore(Val: EventVal, Ptr: EventHandleAddr);
3051 }
3052 // Copy the arguments for outlined function
3053 if (HasShareds) {
3054 Value *Shareds = StaleCI->getArgOperand(i: 1);
3055 Align Alignment = TaskData->getPointerAlignment(DL: M.getDataLayout());
3056 Value *TaskShareds = Builder.CreateLoad(Ty: VoidPtr, Ptr: TaskData);
3057 Builder.CreateMemCpy(Dst: TaskShareds, DstAlign: Alignment, Src: Shareds, SrcAlign: Alignment,
3058 Size: SharedsSize);
3059 }
3060
3061 if (Priority) {
3062 //
3063 // The return type of "__kmpc_omp_task_alloc" is "kmp_task_t *",
3064 // we populate the priority information into the "kmp_task_t" here
3065 //
3066 // The struct "kmp_task_t" definition is available in kmp.h
3067 // kmp_task_t = { shareds, routine, part_id, data1, data2 }
3068 // data2 is used for priority
3069 //
3070 Type *Int32Ty = Builder.getInt32Ty();
3071 Constant *Zero = ConstantInt::get(Ty: Int32Ty, V: 0);
3072 // kmp_task_t* => { ptr }
3073 Type *TaskPtr = StructType::get(elt1: VoidPtr);
3074 Value *TaskGEP =
3075 Builder.CreateInBoundsGEP(Ty: TaskPtr, Ptr: TaskData, IdxList: {Zero, Zero});
3076 // kmp_task_t => { ptr, ptr, i32, ptr, ptr }
3077 Type *TaskStructType = StructType::get(
3078 elt1: VoidPtr, elts: VoidPtr, elts: Builder.getInt32Ty(), elts: VoidPtr, elts: VoidPtr);
3079 Value *PriorityData = Builder.CreateInBoundsGEP(
3080 Ty: TaskStructType, Ptr: TaskGEP, IdxList: {Zero, ConstantInt::get(Ty: Int32Ty, V: 4)});
3081 // kmp_cmplrdata_t => { ptr, ptr }
3082 Type *CmplrStructType = StructType::get(elt1: VoidPtr, elts: VoidPtr);
3083 Value *CmplrData = Builder.CreateInBoundsGEP(Ty: CmplrStructType,
3084 Ptr: PriorityData, IdxList: {Zero, Zero});
3085 Builder.CreateStore(Val: Priority, Ptr: CmplrData);
3086 }
3087
3088 Value *DepArray = nullptr;
3089 Value *NumDeps = nullptr;
3090 if (Dependencies.DepArray) {
3091 DepArray = Dependencies.DepArray;
3092 NumDeps = Dependencies.NumDeps;
3093 } else if (!Dependencies.Deps.empty()) {
3094 DepArray = emitTaskDependencies(OMPBuilder&: *this, Dependencies: Dependencies.Deps);
3095 NumDeps = Builder.getInt32(C: Dependencies.Deps.size());
3096 }
3097
3098 // In the presence of the `if` clause, the following IR is generated:
3099 // ...
3100 // %data = call @__kmpc_omp_task_alloc(...)
3101 // br i1 %if_condition, label %then, label %else
3102 // then:
3103 // call @__kmpc_omp_task(...)
3104 // br label %exit
3105 // else:
3106 // ;; Wait for resolution of dependencies, if any, before
3107 // ;; beginning the task
3108 // call @__kmpc_omp_wait_deps(...)
3109 // call @__kmpc_omp_task_begin_if0(...)
3110 // call @outlined_fn(...)
3111 // call @__kmpc_omp_task_complete_if0(...)
3112 // br label %exit
3113 // exit:
3114 // ...
3115 if (IfCondition && !UseMergedIf0Path) {
3116 // `SplitBlockAndInsertIfThenElse` requires the block to have a
3117 // terminator.
3118 splitBB(Builder, /*CreateBranch=*/true, Name: "if.end");
3119 Instruction *IfTerminator =
3120 Builder.GetInsertPoint()->getParent()->getTerminator();
3121 Instruction *ThenTI = IfTerminator, *ElseTI = nullptr;
3122 Builder.SetInsertPoint(IfTerminator);
3123 SplitBlockAndInsertIfThenElse(Cond: IfCondition, SplitBefore: IfTerminator, ThenTerm: &ThenTI,
3124 ElseTerm: &ElseTI);
3125 Builder.SetInsertPoint(ElseTI);
3126
3127 if (DepArray) {
3128 Function *TaskWaitFn =
3129 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_wait_deps);
3130 createRuntimeFunctionCall(
3131 Callee: TaskWaitFn,
3132 Args: {Ident, ThreadID, NumDeps, DepArray,
3133 ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0),
3134 ConstantPointerNull::get(T: PointerType::getUnqual(C&: M.getContext()))});
3135 }
3136 Function *TaskBeginFn =
3137 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_begin_if0);
3138 Function *TaskCompleteFn =
3139 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_complete_if0);
3140 createRuntimeFunctionCall(Callee: TaskBeginFn, Args: {Ident, ThreadID, TaskData});
3141 CallInst *CI = nullptr;
3142 if (HasShareds)
3143 CI = createRuntimeFunctionCall(Callee: &OutlinedFn, Args: {ThreadID, TaskData});
3144 else
3145 CI = createRuntimeFunctionCall(Callee: &OutlinedFn, Args: {ThreadID});
3146 CI->setDebugLoc(StaleCI->getDebugLoc());
3147 createRuntimeFunctionCall(Callee: TaskCompleteFn, Args: {Ident, ThreadID, TaskData});
3148 Builder.SetInsertPoint(ThenTI);
3149 }
3150
3151 if (DepArray) {
3152 Function *TaskFn =
3153 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_with_deps);
3154 createRuntimeFunctionCall(
3155 Callee: TaskFn,
3156 Args: {Ident, ThreadID, TaskData, NumDeps, DepArray,
3157 ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0),
3158 ConstantPointerNull::get(T: PointerType::getUnqual(C&: M.getContext()))});
3159
3160 } else {
3161 // Emit the @__kmpc_omp_task runtime call to spawn the task
3162 Function *TaskFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task);
3163 createRuntimeFunctionCall(Callee: TaskFn, Args: {Ident, ThreadID, TaskData});
3164 }
3165
3166 StaleCI->eraseFromParent();
3167
3168 Builder.SetInsertPoint(TheBB: TaskAllocaBB, IP: TaskAllocaBB->begin());
3169 if (HasShareds) {
3170 LoadInst *Shareds = Builder.CreateLoad(Ty: VoidPtr, Ptr: OutlinedFn.getArg(i: 1));
3171 OutlinedFn.getArg(i: 1)->replaceUsesWithIf(
3172 New: Shareds, ShouldReplace: [Shareds](Use &U) { return U.getUser() != Shareds; });
3173 }
3174
3175 // The insert point may refer to one of the instructions about to be
3176 // deleted. It is not needed anymore so clear it instead of leaving it
3177 // dangling.
3178 Builder.ClearInsertionPoint();
3179 for (Instruction *I : llvm::reverse(C&: ToBeDeleted))
3180 I->eraseFromParent();
3181 };
3182
3183 addOutlineInfo(OI: std::move(OI));
3184 Builder.SetInsertPoint(TheBB: TaskExitBB, IP: TaskExitBB->begin());
3185
3186 return Builder.saveIP();
3187}
3188
3189OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTaskgroup(
3190 const LocationDescription &Loc, InsertPointTy AllocaIP,
3191 ArrayRef<BasicBlock *> DeallocBlocks, BodyGenCallbackTy BodyGenCB) {
3192 if (!updateToLocation(Loc))
3193 return InsertPointTy();
3194
3195 uint32_t SrcLocStrSize;
3196 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
3197 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
3198 Value *ThreadID = getOrCreateThreadID(Ident);
3199
3200 // Emit the @__kmpc_taskgroup runtime call to start the taskgroup
3201 Function *TaskgroupFn =
3202 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_taskgroup);
3203 createRuntimeFunctionCall(Callee: TaskgroupFn, Args: {Ident, ThreadID});
3204
3205 BasicBlock *TaskgroupExitBB = splitBB(Builder, CreateBranch: true, Name: "taskgroup.exit");
3206 if (Error Err = BodyGenCB(AllocaIP, Builder.saveIP(), DeallocBlocks))
3207 return Err;
3208
3209 Builder.SetInsertPoint(TaskgroupExitBB);
3210 // Emit the @__kmpc_end_taskgroup runtime call to end the taskgroup
3211 Function *EndTaskgroupFn =
3212 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_end_taskgroup);
3213 createRuntimeFunctionCall(Callee: EndTaskgroupFn, Args: {Ident, ThreadID});
3214
3215 return Builder.saveIP();
3216}
3217
3218OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createSections(
3219 const LocationDescription &Loc, InsertPointTy AllocaIP,
3220 ArrayRef<StorableBodyGenCallbackTy> SectionCBs, PrivatizeCallbackTy PrivCB,
3221 FinalizeCallbackTy FiniCB, bool IsCancellable, bool IsNowait) {
3222 assert(!isConflictIP(AllocaIP, Loc.IP) && "Dedicated IP allocas required");
3223
3224 if (!updateToLocation(Loc))
3225 return Loc.IP;
3226
3227 FinalizationStack.push_back(Elt: {FiniCB, OMPD_sections, IsCancellable});
3228
3229 // Each section is emitted as a switch case
3230 // Each finalization callback is handled from clang.EmitOMPSectionDirective()
3231 // -> OMP.createSection() which generates the IR for each section
3232 // Iterate through all sections and emit a switch construct:
3233 // switch (IV) {
3234 // case 0:
3235 // <SectionStmt[0]>;
3236 // break;
3237 // ...
3238 // case <NumSection> - 1:
3239 // <SectionStmt[<NumSection> - 1]>;
3240 // break;
3241 // }
3242 // ...
3243 // section_loop.after:
3244 // <FiniCB>;
3245 auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, Value *IndVar) -> Error {
3246 Builder.restoreIP(IP: CodeGenIP);
3247 BasicBlock *Continue =
3248 splitBBWithSuffix(Builder, /*CreateBranch=*/false, Suffix: ".sections.after");
3249 Function *CurFn = Continue->getParent();
3250 SwitchInst *SwitchStmt = Builder.CreateSwitch(V: IndVar, Dest: Continue);
3251
3252 unsigned CaseNumber = 0;
3253 for (auto SectionCB : SectionCBs) {
3254 BasicBlock *CaseBB = BasicBlock::Create(
3255 Context&: M.getContext(), Name: "omp_section_loop.body.case", Parent: CurFn, InsertBefore: Continue);
3256 SwitchStmt->addCase(OnVal: Builder.getInt32(C: CaseNumber), Dest: CaseBB);
3257 Builder.SetInsertPoint(CaseBB);
3258 UncondBrInst *CaseEndBr = Builder.CreateBr(Dest: Continue);
3259 if (Error Err =
3260 SectionCB(InsertPointTy(),
3261 {CaseEndBr->getParent(), CaseEndBr->getIterator()}, {}))
3262 return Err;
3263 CaseNumber++;
3264 }
3265 // remove the existing terminator from body BB since there can be no
3266 // terminators after switch/case
3267 return Error::success();
3268 };
3269 // Loop body ends here
3270 // LowerBound, UpperBound, and STride for createCanonicalLoop
3271 Type *I32Ty = Type::getInt32Ty(C&: M.getContext());
3272 Value *LB = ConstantInt::get(Ty: I32Ty, V: 0);
3273 Value *UB = ConstantInt::get(Ty: I32Ty, V: SectionCBs.size());
3274 Value *ST = ConstantInt::get(Ty: I32Ty, V: 1);
3275 Expected<CanonicalLoopInfo *> LoopInfo = createCanonicalLoop(
3276 Loc, BodyGenCB: LoopBodyGenCB, Start: LB, Stop: UB, Step: ST, IsSigned: true, InclusiveStop: false, ComputeIP: AllocaIP, Name: "section_loop");
3277 if (!LoopInfo)
3278 return LoopInfo.takeError();
3279
3280 InsertPointOrErrorTy WsloopIP =
3281 applyStaticWorkshareLoop(DL: Loc.DL, CLI: *LoopInfo, AllocaIP,
3282 LoopType: WorksharingLoopType::ForStaticLoop, NeedsBarrier: !IsNowait);
3283 if (!WsloopIP)
3284 return WsloopIP.takeError();
3285 InsertPointTy AfterIP = *WsloopIP;
3286
3287 BasicBlock *LoopFini = AfterIP.getBlock()->getSinglePredecessor();
3288 assert(LoopFini && "Bad structure of static workshare loop finalization");
3289
3290 // Apply the finalization callback in LoopAfterBB
3291 auto FiniInfo = FinalizationStack.pop_back_val();
3292 assert(FiniInfo.DK == OMPD_sections &&
3293 "Unexpected finalization stack state!");
3294 if (Error Err = FiniInfo.mergeFiniBB(Builder, OtherFiniBB: LoopFini))
3295 return Err;
3296
3297 return AfterIP;
3298}
3299
3300OpenMPIRBuilder::InsertPointOrErrorTy
3301OpenMPIRBuilder::createSection(const LocationDescription &Loc,
3302 BodyGenCallbackTy BodyGenCB,
3303 FinalizeCallbackTy FiniCB) {
3304 if (!updateToLocation(Loc))
3305 return Loc.IP;
3306
3307 auto FiniCBWrapper = [&](InsertPointTy IP) {
3308 if (IP.getBlock()->end() != IP.getPoint())
3309 return FiniCB(IP);
3310 // This must be done otherwise any nested constructs using FinalizeOMPRegion
3311 // will fail because that function requires the Finalization Basic Block to
3312 // have a terminator, which is already removed by EmitOMPRegionBody.
3313 // IP is currently at cancelation block.
3314 // We need to backtrack to the condition block to fetch
3315 // the exit block and create a branch from cancelation
3316 // to exit block.
3317 IRBuilder<>::InsertPointGuard IPG(Builder);
3318 Builder.restoreIP(IP);
3319 auto *CaseBB = Loc.IP.getBlock();
3320 auto *CondBB = CaseBB->getSinglePredecessor()->getSinglePredecessor();
3321 auto *ExitBB = CondBB->getTerminator()->getSuccessor(Idx: 1);
3322 Instruction *I = Builder.CreateBr(Dest: ExitBB);
3323 IP = InsertPointTy(I->getParent(), I->getIterator());
3324 return FiniCB(IP);
3325 };
3326
3327 Directive OMPD = Directive::OMPD_sections;
3328 // Since we are using Finalization Callback here, HasFinalize
3329 // and IsCancellable have to be true
3330 return EmitOMPInlinedRegion(OMPD, EntryCall: nullptr, ExitCall: nullptr, BodyGenCB, FiniCB: FiniCBWrapper,
3331 /*Conditional*/ false, /*hasFinalize*/ HasFinalize: true,
3332 /*IsCancellable*/ true);
3333}
3334
3335static OpenMPIRBuilder::InsertPointTy getInsertPointAfterInstr(Instruction *I) {
3336 BasicBlock::iterator IT(I);
3337 IT++;
3338 return OpenMPIRBuilder::InsertPointTy(I->getParent(), IT);
3339}
3340
3341Value *OpenMPIRBuilder::getGPUThreadID() {
3342 return createRuntimeFunctionCall(
3343 Callee: getOrCreateRuntimeFunction(M,
3344 FnID: OMPRTL___kmpc_get_hardware_thread_id_in_block),
3345 Args: {});
3346}
3347
3348Value *OpenMPIRBuilder::getGPUWarpSize() {
3349 return createRuntimeFunctionCall(
3350 Callee: getOrCreateRuntimeFunction(M, FnID: OMPRTL___kmpc_get_warp_size), Args: {});
3351}
3352
3353Value *OpenMPIRBuilder::getNVPTXWarpID() {
3354 unsigned LaneIDBits = Log2_32(Value: Config.getGridValue().GV_Warp_Size);
3355 return Builder.CreateAShr(LHS: getGPUThreadID(), RHS: LaneIDBits, Name: "nvptx_warp_id");
3356}
3357
3358Value *OpenMPIRBuilder::getNVPTXLaneID() {
3359 unsigned LaneIDBits = Log2_32(Value: Config.getGridValue().GV_Warp_Size);
3360 assert(LaneIDBits < 32 && "Invalid LaneIDBits size in NVPTX device.");
3361 unsigned LaneIDMask = ~0u >> (32u - LaneIDBits);
3362 return Builder.CreateAnd(LHS: getGPUThreadID(), RHS: Builder.getInt32(C: LaneIDMask),
3363 Name: "nvptx_lane_id");
3364}
3365
3366Value *OpenMPIRBuilder::castValueToType(InsertPointTy AllocaIP, Value *From,
3367 Type *ToType) {
3368 Type *FromType = From->getType();
3369 uint64_t FromSize = M.getDataLayout().getTypeStoreSize(Ty: FromType);
3370 uint64_t ToSize = M.getDataLayout().getTypeStoreSize(Ty: ToType);
3371 assert(FromSize > 0 && "From size must be greater than zero");
3372 assert(ToSize > 0 && "To size must be greater than zero");
3373 if (FromType == ToType)
3374 return From;
3375 if (FromSize == ToSize)
3376 return Builder.CreateBitCast(V: From, DestTy: ToType);
3377 if (ToType->isIntegerTy() && FromType->isIntegerTy())
3378 return Builder.CreateIntCast(V: From, DestTy: ToType, /*isSigned*/ true);
3379 InsertPointTy SaveIP = Builder.saveIP();
3380 Builder.restoreIP(IP: AllocaIP);
3381 Value *CastItem = Builder.CreateAlloca(Ty: ToType);
3382 Builder.restoreIP(IP: SaveIP);
3383
3384 Value *ValCastItem = Builder.CreatePointerBitCastOrAddrSpaceCast(
3385 V: CastItem, DestTy: Builder.getPtrTy(AddrSpace: 0));
3386 Builder.CreateStore(Val: From, Ptr: ValCastItem);
3387 return Builder.CreateLoad(Ty: ToType, Ptr: CastItem);
3388}
3389
3390Value *OpenMPIRBuilder::createRuntimeShuffleFunction(InsertPointTy AllocaIP,
3391 Value *Element,
3392 Type *ElementType,
3393 Value *Offset) {
3394 uint64_t Size = M.getDataLayout().getTypeStoreSize(Ty: ElementType);
3395 assert(Size <= 8 && "Unsupported bitwidth in shuffle instruction");
3396
3397 // Cast all types to 32- or 64-bit values before calling shuffle routines.
3398 Type *CastTy = Builder.getIntNTy(N: Size <= 4 ? 32 : 64);
3399 Value *ElemCast = castValueToType(AllocaIP, From: Element, ToType: CastTy);
3400 Value *WarpSize =
3401 Builder.CreateIntCast(V: getGPUWarpSize(), DestTy: Builder.getInt16Ty(), isSigned: true);
3402 Function *ShuffleFunc = getOrCreateRuntimeFunctionPtr(
3403 FnID: Size <= 4 ? RuntimeFunction::OMPRTL___kmpc_shuffle_int32
3404 : RuntimeFunction::OMPRTL___kmpc_shuffle_int64);
3405 Value *WarpSizeCast =
3406 Builder.CreateIntCast(V: WarpSize, DestTy: Builder.getInt16Ty(), /*isSigned=*/true);
3407 Value *ShuffleCall =
3408 createRuntimeFunctionCall(Callee: ShuffleFunc, Args: {ElemCast, Offset, WarpSizeCast});
3409 // The shuffle runtime functions return a 32- or 64-bit value. Cast it back
3410 // down to the requested element type, otherwise storing the result would
3411 // write past the end of an element narrower than the shuffle width.
3412 return castValueToType(AllocaIP, From: ShuffleCall, ToType: ElementType);
3413}
3414
3415void OpenMPIRBuilder::shuffleAndStore(InsertPointTy AllocaIP, Value *SrcAddr,
3416 Value *DstAddr, Type *ElemType,
3417 Value *Offset, Type *ReductionArrayTy,
3418 bool IsByRefElem) {
3419 uint64_t Size = M.getDataLayout().getTypeStoreSize(Ty: ElemType);
3420 // Create the loop over the big sized data.
3421 // ptr = (void*)Elem;
3422 // ptrEnd = (void*) Elem + 1;
3423 // Step = 8;
3424 // while (ptr + Step < ptrEnd)
3425 // shuffle((int64_t)*ptr);
3426 // Step = 4;
3427 // while (ptr + Step < ptrEnd)
3428 // shuffle((int32_t)*ptr);
3429 // ...
3430 Type *IndexTy = Builder.getIndexTy(
3431 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
3432 Value *ElemPtr = DstAddr;
3433 Value *Ptr = SrcAddr;
3434 for (unsigned IntSize = 8; IntSize >= 1; IntSize /= 2) {
3435 if (Size < IntSize)
3436 continue;
3437 Type *IntType = Builder.getIntNTy(N: IntSize * 8);
3438 Ptr = Builder.CreatePointerBitCastOrAddrSpaceCast(
3439 V: Ptr, DestTy: Builder.getPtrTy(AddrSpace: 0), Name: Ptr->getName() + ".ascast");
3440 Value *SrcAddrGEP =
3441 Builder.CreateGEP(Ty: ElemType, Ptr: SrcAddr, IdxList: {ConstantInt::get(Ty: IndexTy, V: 1)});
3442 ElemPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
3443 V: ElemPtr, DestTy: Builder.getPtrTy(AddrSpace: 0), Name: ElemPtr->getName() + ".ascast");
3444
3445 Function *CurFunc = Builder.GetInsertBlock()->getParent();
3446 if ((Size / IntSize) > 1) {
3447 Value *PtrEnd = Builder.CreatePointerBitCastOrAddrSpaceCast(
3448 V: SrcAddrGEP, DestTy: Builder.getPtrTy());
3449 BasicBlock *PreCondBB =
3450 BasicBlock::Create(Context&: M.getContext(), Name: ".shuffle.pre_cond");
3451 BasicBlock *ThenBB = BasicBlock::Create(Context&: M.getContext(), Name: ".shuffle.then");
3452 BasicBlock *ExitBB = BasicBlock::Create(Context&: M.getContext(), Name: ".shuffle.exit");
3453 BasicBlock *CurrentBB = Builder.GetInsertBlock();
3454 emitBlock(BB: PreCondBB, CurFn: CurFunc);
3455 PHINode *PhiSrc =
3456 Builder.CreatePHI(Ty: Ptr->getType(), /*NumReservedValues=*/2);
3457 PhiSrc->addIncoming(V: Ptr, BB: CurrentBB);
3458 PHINode *PhiDest =
3459 Builder.CreatePHI(Ty: ElemPtr->getType(), /*NumReservedValues=*/2);
3460 PhiDest->addIncoming(V: ElemPtr, BB: CurrentBB);
3461 Ptr = PhiSrc;
3462 ElemPtr = PhiDest;
3463 Value *PtrDiff = Builder.CreatePtrDiff(
3464 ElemTy: Builder.getInt8Ty(), LHS: PtrEnd,
3465 RHS: Builder.CreatePointerBitCastOrAddrSpaceCast(V: Ptr, DestTy: Builder.getPtrTy()));
3466 Builder.CreateCondBr(
3467 Cond: Builder.CreateICmpSGT(LHS: PtrDiff, RHS: Builder.getInt64(C: IntSize - 1)), True: ThenBB,
3468 False: ExitBB);
3469 emitBlock(BB: ThenBB, CurFn: CurFunc);
3470 Value *Res = createRuntimeShuffleFunction(
3471 AllocaIP,
3472 Element: Builder.CreateAlignedLoad(
3473 Ty: IntType, Ptr, Align: M.getDataLayout().getPrefTypeAlign(Ty: ElemType)),
3474 ElementType: IntType, Offset);
3475 Builder.CreateAlignedStore(Val: Res, Ptr: ElemPtr,
3476 Align: M.getDataLayout().getPrefTypeAlign(Ty: ElemType));
3477 Value *LocalPtr =
3478 Builder.CreateGEP(Ty: IntType, Ptr, IdxList: {ConstantInt::get(Ty: IndexTy, V: 1)});
3479 Value *LocalElemPtr =
3480 Builder.CreateGEP(Ty: IntType, Ptr: ElemPtr, IdxList: {ConstantInt::get(Ty: IndexTy, V: 1)});
3481 PhiSrc->addIncoming(V: LocalPtr, BB: ThenBB);
3482 PhiDest->addIncoming(V: LocalElemPtr, BB: ThenBB);
3483 emitBranch(Target: PreCondBB);
3484 emitBlock(BB: ExitBB, CurFn: CurFunc);
3485 } else {
3486 // The shuffled value comes back as the chunk's integer type, so the
3487 // store covers exactly this chunk regardless of what ElemType is.
3488 Value *Res = createRuntimeShuffleFunction(
3489 AllocaIP, Element: Builder.CreateLoad(Ty: IntType, Ptr), ElementType: IntType, Offset);
3490 Builder.CreateStore(Val: Res, Ptr: ElemPtr);
3491 Ptr = Builder.CreateGEP(Ty: IntType, Ptr, IdxList: {ConstantInt::get(Ty: IndexTy, V: 1)});
3492 ElemPtr =
3493 Builder.CreateGEP(Ty: IntType, Ptr: ElemPtr, IdxList: {ConstantInt::get(Ty: IndexTy, V: 1)});
3494 }
3495 Size = Size % IntSize;
3496 }
3497}
3498
3499Error OpenMPIRBuilder::emitReductionListCopy(
3500 InsertPointTy AllocaIP, CopyAction Action, Type *ReductionArrayTy,
3501 ArrayRef<ReductionInfo> ReductionInfos, Value *SrcBase, Value *DestBase,
3502 ArrayRef<bool> IsByRef, CopyOptionsTy CopyOptions) {
3503 Type *IndexTy = Builder.getIndexTy(
3504 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
3505 Value *RemoteLaneOffset = CopyOptions.RemoteLaneOffset;
3506
3507 // Iterates, element-by-element, through the source Reduce list and
3508 // make a copy.
3509 for (auto En : enumerate(First&: ReductionInfos)) {
3510 const ReductionInfo &RI = En.value();
3511 Value *SrcElementAddr = nullptr;
3512 AllocaInst *DestAlloca = nullptr;
3513 Value *DestElementAddr = nullptr;
3514 Value *DestElementPtrAddr = nullptr;
3515 // Should we shuffle in an element from a remote lane?
3516 bool ShuffleInElement = false;
3517 // Set to true to update the pointer in the dest Reduce list to a
3518 // newly created element.
3519 bool UpdateDestListPtr = false;
3520
3521 // Step 1.1: Get the address for the src element in the Reduce list.
3522 Value *SrcElementPtrAddr = Builder.CreateInBoundsGEP(
3523 Ty: ReductionArrayTy, Ptr: SrcBase,
3524 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
3525 SrcElementAddr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: SrcElementPtrAddr);
3526
3527 // Step 1.2: Create a temporary to store the element in the destination
3528 // Reduce list.
3529 DestElementPtrAddr = Builder.CreateInBoundsGEP(
3530 Ty: ReductionArrayTy, Ptr: DestBase,
3531 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
3532 bool IsByRefElem = (!IsByRef.empty() && IsByRef[En.index()]);
3533 switch (Action) {
3534 case CopyAction::RemoteLaneToThread: {
3535 InsertPointTy CurIP = Builder.saveIP();
3536 Builder.restoreIP(IP: AllocaIP);
3537
3538 Type *DestAllocaType =
3539 IsByRefElem ? RI.ByRefAllocatedType : RI.ElementType;
3540 DestAlloca = Builder.CreateAlloca(Ty: DestAllocaType, ArraySize: nullptr,
3541 Name: ".omp.reduction.element");
3542 DestAlloca->setAlignment(
3543 M.getDataLayout().getPrefTypeAlign(Ty: DestAllocaType));
3544 DestElementAddr = DestAlloca;
3545 DestElementAddr =
3546 Builder.CreateAddrSpaceCast(V: DestElementAddr, DestTy: Builder.getPtrTy(),
3547 Name: DestElementAddr->getName() + ".ascast");
3548 Builder.restoreIP(IP: CurIP);
3549 ShuffleInElement = true;
3550 UpdateDestListPtr = true;
3551 break;
3552 }
3553 case CopyAction::ThreadCopy: {
3554 DestElementAddr =
3555 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: DestElementPtrAddr);
3556 break;
3557 }
3558 }
3559
3560 // Now that all active lanes have read the element in the
3561 // Reduce list, shuffle over the value from the remote lane.
3562 if (ShuffleInElement) {
3563 Type *ShuffleType = RI.ElementType;
3564 Value *ShuffleSrcAddr = SrcElementAddr;
3565 Value *ShuffleDestAddr = DestElementAddr;
3566 AllocaInst *LocalStorage = nullptr;
3567
3568 if (IsByRefElem) {
3569 assert(RI.ByRefElementType && "Expected by-ref element type to be set");
3570 assert(RI.ByRefAllocatedType &&
3571 "Expected by-ref allocated type to be set");
3572 // For by-ref reductions, we need to copy from the remote lane the
3573 // actual value of the partial reduction computed by that remote lane;
3574 // rather than, for example, a pointer to that data or, even worse, a
3575 // pointer to the descriptor of the by-ref reduction element.
3576 ShuffleType = RI.ByRefElementType;
3577
3578 if (RI.DataPtrPtrGen) {
3579 // Descriptor-based by-ref: extract data pointer from descriptor.
3580 InsertPointOrErrorTy GenResult = RI.DataPtrPtrGen(
3581 Builder.saveIP(), ShuffleSrcAddr, ShuffleSrcAddr);
3582
3583 if (!GenResult)
3584 return GenResult.takeError();
3585
3586 ShuffleSrcAddr =
3587 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ShuffleSrcAddr);
3588
3589 {
3590 InsertPointTy OldIP = Builder.saveIP();
3591 Builder.restoreIP(IP: AllocaIP);
3592
3593 LocalStorage = Builder.CreateAlloca(Ty: ShuffleType);
3594 Builder.restoreIP(IP: OldIP);
3595 ShuffleDestAddr = LocalStorage;
3596 }
3597 } else {
3598 // Non-descriptor by-ref: the pointer already references data
3599 // directly. Shuffle into the destination alloca.
3600 ShuffleDestAddr = DestElementAddr;
3601 }
3602 }
3603
3604 shuffleAndStore(AllocaIP, SrcAddr: ShuffleSrcAddr, DstAddr: ShuffleDestAddr, ElemType: ShuffleType,
3605 Offset: RemoteLaneOffset, ReductionArrayTy, IsByRefElem);
3606
3607 if (IsByRefElem && RI.DataPtrPtrGen) {
3608 // Copy descriptor from source and update base_ptr to shuffled data
3609 Value *DestDescriptorAddr = Builder.CreatePointerBitCastOrAddrSpaceCast(
3610 V: DestAlloca, DestTy: Builder.getPtrTy(), Name: ".ascast");
3611
3612 InsertPointOrErrorTy GenResult = generateReductionDescriptor(
3613 DescriptorAddr: DestDescriptorAddr, DataPtr: LocalStorage, SrcDescriptorAddr: SrcElementAddr,
3614 DescriptorType: RI.ByRefAllocatedType, DataPtrPtrGen: RI.DataPtrPtrGen);
3615
3616 if (!GenResult)
3617 return GenResult.takeError();
3618 }
3619 } else {
3620 switch (RI.EvaluationKind) {
3621 case EvalKind::Scalar: {
3622 Value *Elem = Builder.CreateLoad(Ty: RI.ElementType, Ptr: SrcElementAddr);
3623 // Store the source element value to the dest element address.
3624 Builder.CreateStore(Val: Elem, Ptr: DestElementAddr);
3625 break;
3626 }
3627 case EvalKind::Complex: {
3628 Value *SrcRealPtr = Builder.CreateConstInBoundsGEP2_32(
3629 Ty: RI.ElementType, Ptr: SrcElementAddr, Idx0: 0, Idx1: 0, Name: ".realp");
3630 Value *SrcReal = Builder.CreateLoad(
3631 Ty: RI.ElementType->getStructElementType(N: 0), Ptr: SrcRealPtr, Name: ".real");
3632 Value *SrcImgPtr = Builder.CreateConstInBoundsGEP2_32(
3633 Ty: RI.ElementType, Ptr: SrcElementAddr, Idx0: 0, Idx1: 1, Name: ".imagp");
3634 Value *SrcImg = Builder.CreateLoad(
3635 Ty: RI.ElementType->getStructElementType(N: 1), Ptr: SrcImgPtr, Name: ".imag");
3636
3637 Value *DestRealPtr = Builder.CreateConstInBoundsGEP2_32(
3638 Ty: RI.ElementType, Ptr: DestElementAddr, Idx0: 0, Idx1: 0, Name: ".realp");
3639 Value *DestImgPtr = Builder.CreateConstInBoundsGEP2_32(
3640 Ty: RI.ElementType, Ptr: DestElementAddr, Idx0: 0, Idx1: 1, Name: ".imagp");
3641 Builder.CreateStore(Val: SrcReal, Ptr: DestRealPtr);
3642 Builder.CreateStore(Val: SrcImg, Ptr: DestImgPtr);
3643 break;
3644 }
3645 case EvalKind::Aggregate: {
3646 Value *SizeVal = Builder.getInt64(
3647 C: M.getDataLayout().getTypeStoreSize(Ty: RI.ElementType));
3648 Builder.CreateMemCpy(
3649 Dst: DestElementAddr, DstAlign: M.getDataLayout().getPrefTypeAlign(Ty: RI.ElementType),
3650 Src: SrcElementAddr, SrcAlign: M.getDataLayout().getPrefTypeAlign(Ty: RI.ElementType),
3651 Size: SizeVal, isVolatile: false);
3652 break;
3653 }
3654 };
3655 }
3656
3657 // Step 3.1: Modify reference in dest Reduce list as needed.
3658 // Modifying the reference in Reduce list to point to the newly
3659 // created element. The element is live in the current function
3660 // scope and that of functions it invokes (i.e., reduce_function).
3661 // RemoteReduceData[i] = (void*)&RemoteElem
3662 if (UpdateDestListPtr) {
3663 Value *CastDestAddr = Builder.CreatePointerBitCastOrAddrSpaceCast(
3664 V: DestElementAddr, DestTy: Builder.getPtrTy(),
3665 Name: DestElementAddr->getName() + ".ascast");
3666 Builder.CreateStore(Val: CastDestAddr, Ptr: DestElementPtrAddr);
3667 }
3668 }
3669
3670 return Error::success();
3671}
3672
3673Expected<Function *> OpenMPIRBuilder::emitInterWarpCopyFunction(
3674 const LocationDescription &Loc, ArrayRef<ReductionInfo> ReductionInfos,
3675 AttributeList FuncAttrs, ArrayRef<bool> IsByRef) {
3676 IRBuilder<>::InsertPointGuard IPG(Builder);
3677 LLVMContext &Ctx = M.getContext();
3678 FunctionType *FuncTy = FunctionType::get(
3679 Result: Builder.getVoidTy(), Params: {Builder.getPtrTy(), Builder.getInt32Ty()},
3680 /* IsVarArg */ isVarArg: false);
3681 Function *WcFunc =
3682 Function::Create(Ty: FuncTy, Linkage: GlobalVariable::InternalLinkage,
3683 N: "_omp_reduction_inter_warp_copy_func", M: &M);
3684 WcFunc->setCallingConv(Config.getRuntimeCC());
3685 WcFunc->setAttributes(FuncAttrs);
3686 WcFunc->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
3687 WcFunc->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
3688 BasicBlock *EntryBB = BasicBlock::Create(Context&: M.getContext(), Name: "entry", Parent: WcFunc);
3689 Builder.SetInsertPoint(EntryBB);
3690 Builder.SetCurrentDebugLocation(llvm::DebugLoc());
3691
3692 // ReduceList: thread local Reduce list.
3693 // At the stage of the computation when this function is called, partially
3694 // aggregated values reside in the first lane of every active warp.
3695 Argument *ReduceListArg = WcFunc->getArg(i: 0);
3696 // NumWarps: number of warps active in the parallel region. This could
3697 // be smaller than 32 (max warps in a CTA) for partial block reduction.
3698 Argument *NumWarpsArg = WcFunc->getArg(i: 1);
3699
3700 // This array is used as a medium to transfer, one reduce element at a time,
3701 // the data from the first lane of every warp to lanes in the first warp
3702 // in order to perform the final step of a reduction in a parallel region
3703 // (reduction across warps). The array is placed in NVPTX __shared__ memory
3704 // for reduced latency, as well as to have a distinct copy for concurrently
3705 // executing target regions. The array is declared with common linkage so
3706 // as to be shared across compilation units.
3707 StringRef TransferMediumName =
3708 "__openmp_nvptx_data_transfer_temporary_storage";
3709 GlobalVariable *TransferMedium = M.getGlobalVariable(Name: TransferMediumName);
3710 unsigned WarpSize = Config.getGridValue().GV_Warp_Size;
3711 ArrayType *ArrayTy = ArrayType::get(ElementType: Builder.getInt32Ty(), NumElements: WarpSize);
3712 if (!TransferMedium) {
3713 TransferMedium = new GlobalVariable(
3714 M, ArrayTy, /*isConstant=*/false, GlobalVariable::WeakAnyLinkage,
3715 UndefValue::get(T: ArrayTy), TransferMediumName,
3716 /*InsertBefore=*/nullptr, GlobalVariable::NotThreadLocal,
3717 /*AddressSpace=*/3);
3718 }
3719
3720 // Get the CUDA thread id of the current OpenMP thread on the GPU.
3721 Value *GPUThreadID = getGPUThreadID();
3722 // nvptx_lane_id = nvptx_id % warpsize
3723 Value *LaneID = getNVPTXLaneID();
3724 // nvptx_warp_id = nvptx_id / warpsize
3725 Value *WarpID = getNVPTXWarpID();
3726
3727 InsertPointTy AllocaIP =
3728 InsertPointTy(Builder.GetInsertBlock(),
3729 Builder.GetInsertBlock()->getFirstInsertionPt());
3730 Type *Arg0Type = ReduceListArg->getType();
3731 Type *Arg1Type = NumWarpsArg->getType();
3732 Builder.restoreIP(IP: AllocaIP);
3733 AllocaInst *ReduceListAlloca = Builder.CreateAlloca(
3734 Ty: Arg0Type, ArraySize: nullptr, Name: ReduceListArg->getName() + ".addr");
3735 AllocaInst *NumWarpsAlloca =
3736 Builder.CreateAlloca(Ty: Arg1Type, ArraySize: nullptr, Name: NumWarpsArg->getName() + ".addr");
3737 Value *ReduceListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
3738 V: ReduceListAlloca, DestTy: Arg0Type, Name: ReduceListAlloca->getName() + ".ascast");
3739 Value *NumWarpsAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
3740 V: NumWarpsAlloca, DestTy: Builder.getPtrTy(AddrSpace: 0),
3741 Name: NumWarpsAlloca->getName() + ".ascast");
3742 Builder.CreateStore(Val: ReduceListArg, Ptr: ReduceListAddrCast);
3743 Builder.CreateStore(Val: NumWarpsArg, Ptr: NumWarpsAddrCast);
3744 AllocaIP = getInsertPointAfterInstr(I: NumWarpsAlloca);
3745 InsertPointTy CodeGenIP =
3746 getInsertPointAfterInstr(I: &Builder.GetInsertBlock()->back());
3747 Builder.restoreIP(IP: CodeGenIP);
3748
3749 Value *ReduceList =
3750 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ReduceListAddrCast);
3751
3752 for (auto En : enumerate(First&: ReductionInfos)) {
3753 //
3754 // Warp master copies reduce element to transfer medium in __shared__
3755 // memory.
3756 //
3757 const ReductionInfo &RI = En.value();
3758 bool IsByRefElem = !IsByRef.empty() && IsByRef[En.index()];
3759 unsigned RealTySize = M.getDataLayout().getTypeAllocSize(
3760 Ty: IsByRefElem ? RI.ByRefElementType : RI.ElementType);
3761 for (unsigned TySize = 4; TySize > 0 && RealTySize > 0; TySize /= 2) {
3762 Type *CType = Builder.getIntNTy(N: TySize * 8);
3763
3764 unsigned NumIters = RealTySize / TySize;
3765 if (NumIters == 0)
3766 continue;
3767 Value *Cnt = nullptr;
3768 Value *CntAddr = nullptr;
3769 BasicBlock *PrecondBB = nullptr;
3770 BasicBlock *ExitBB = nullptr;
3771 if (NumIters > 1) {
3772 CodeGenIP = Builder.saveIP();
3773 Builder.restoreIP(IP: AllocaIP);
3774 CntAddr =
3775 Builder.CreateAlloca(Ty: Builder.getInt32Ty(), ArraySize: nullptr, Name: ".cnt.addr");
3776
3777 CntAddr = Builder.CreateAddrSpaceCast(V: CntAddr, DestTy: Builder.getPtrTy(),
3778 Name: CntAddr->getName() + ".ascast");
3779 Builder.restoreIP(IP: CodeGenIP);
3780 Builder.CreateStore(Val: Constant::getNullValue(Ty: Builder.getInt32Ty()),
3781 Ptr: CntAddr,
3782 /*Volatile=*/isVolatile: false);
3783 PrecondBB = BasicBlock::Create(Context&: Ctx, Name: "precond");
3784 ExitBB = BasicBlock::Create(Context&: Ctx, Name: "exit");
3785 BasicBlock *BodyBB = BasicBlock::Create(Context&: Ctx, Name: "body");
3786 emitBlock(BB: PrecondBB, CurFn: Builder.GetInsertBlock()->getParent());
3787 Cnt = Builder.CreateLoad(Ty: Builder.getInt32Ty(), Ptr: CntAddr,
3788 /*Volatile=*/isVolatile: false);
3789 Value *Cmp = Builder.CreateICmpULT(
3790 LHS: Cnt, RHS: ConstantInt::get(Ty: Builder.getInt32Ty(), V: NumIters));
3791 Builder.CreateCondBr(Cond: Cmp, True: BodyBB, False: ExitBB);
3792 emitBlock(BB: BodyBB, CurFn: Builder.GetInsertBlock()->getParent());
3793 }
3794
3795 // kmpc_barrier.
3796 InsertPointOrErrorTy BarrierIP1 =
3797 createBarrier(Loc: LocationDescription(Builder.saveIP(), DebugLoc()),
3798 Kind: omp::Directive::OMPD_unknown,
3799 /* ForceSimpleCall */ false,
3800 /* CheckCancelFlag */ true);
3801 if (!BarrierIP1)
3802 return BarrierIP1.takeError();
3803 BasicBlock *ThenBB = BasicBlock::Create(Context&: Ctx, Name: "then");
3804 BasicBlock *ElseBB = BasicBlock::Create(Context&: Ctx, Name: "else");
3805 BasicBlock *MergeBB = BasicBlock::Create(Context&: Ctx, Name: "ifcont");
3806
3807 // if (lane_id == 0)
3808 Value *IsWarpMaster = Builder.CreateIsNull(Arg: LaneID, Name: "warp_master");
3809 Builder.CreateCondBr(Cond: IsWarpMaster, True: ThenBB, False: ElseBB);
3810 emitBlock(BB: ThenBB, CurFn: Builder.GetInsertBlock()->getParent());
3811
3812 // Reduce element = LocalReduceList[i]
3813 auto *RedListArrayTy =
3814 ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: ReductionInfos.size());
3815 Type *IndexTy = Builder.getIndexTy(
3816 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
3817 Value *ElemPtrPtr =
3818 Builder.CreateInBoundsGEP(Ty: RedListArrayTy, Ptr: ReduceList,
3819 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0),
3820 ConstantInt::get(Ty: IndexTy, V: En.index())});
3821 // elemptr = ((CopyType*)(elemptrptr)) + I
3822 Value *ElemPtr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ElemPtrPtr);
3823
3824 if (IsByRefElem && RI.DataPtrPtrGen) {
3825 InsertPointOrErrorTy GenRes =
3826 RI.DataPtrPtrGen(Builder.saveIP(), ElemPtr, ElemPtr);
3827
3828 if (!GenRes)
3829 return GenRes.takeError();
3830
3831 ElemPtr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ElemPtr);
3832 }
3833
3834 if (NumIters > 1)
3835 ElemPtr = Builder.CreateGEP(Ty: Builder.getInt32Ty(), Ptr: ElemPtr, IdxList: Cnt);
3836
3837 // Get pointer to location in transfer medium.
3838 // MediumPtr = &medium[warp_id]
3839 Value *MediumPtr = Builder.CreateInBoundsGEP(
3840 Ty: ArrayTy, Ptr: TransferMedium, IdxList: {Builder.getInt64(C: 0), WarpID});
3841 // elem = *elemptr
3842 //*MediumPtr = elem
3843 Value *Elem = Builder.CreateLoad(Ty: CType, Ptr: ElemPtr);
3844 // Store the source element value to the dest element address.
3845 Builder.CreateStore(Val: Elem, Ptr: MediumPtr,
3846 /*IsVolatile*/ isVolatile: true);
3847 Builder.CreateBr(Dest: MergeBB);
3848
3849 // else
3850 emitBlock(BB: ElseBB, CurFn: Builder.GetInsertBlock()->getParent());
3851 Builder.CreateBr(Dest: MergeBB);
3852
3853 // endif
3854 emitBlock(BB: MergeBB, CurFn: Builder.GetInsertBlock()->getParent());
3855 InsertPointOrErrorTy BarrierIP2 =
3856 createBarrier(Loc: LocationDescription(Builder.saveIP(), DebugLoc()),
3857 Kind: omp::Directive::OMPD_unknown,
3858 /* ForceSimpleCall */ false,
3859 /* CheckCancelFlag */ true);
3860 if (!BarrierIP2)
3861 return BarrierIP2.takeError();
3862
3863 // Warp 0 copies reduce element from transfer medium
3864 BasicBlock *W0ThenBB = BasicBlock::Create(Context&: Ctx, Name: "then");
3865 BasicBlock *W0ElseBB = BasicBlock::Create(Context&: Ctx, Name: "else");
3866 BasicBlock *W0MergeBB = BasicBlock::Create(Context&: Ctx, Name: "ifcont");
3867
3868 Value *NumWarpsVal =
3869 Builder.CreateLoad(Ty: Builder.getInt32Ty(), Ptr: NumWarpsAddrCast);
3870 // Up to 32 threads in warp 0 are active.
3871 Value *IsActiveThread =
3872 Builder.CreateICmpULT(LHS: GPUThreadID, RHS: NumWarpsVal, Name: "is_active_thread");
3873 Builder.CreateCondBr(Cond: IsActiveThread, True: W0ThenBB, False: W0ElseBB);
3874
3875 emitBlock(BB: W0ThenBB, CurFn: Builder.GetInsertBlock()->getParent());
3876
3877 // SecMediumPtr = &medium[tid]
3878 // SrcMediumVal = *SrcMediumPtr
3879 Value *SrcMediumPtrVal = Builder.CreateInBoundsGEP(
3880 Ty: ArrayTy, Ptr: TransferMedium, IdxList: {Builder.getInt64(C: 0), GPUThreadID});
3881 // TargetElemPtr = (CopyType*)(SrcDataAddr[i]) + I
3882 Value *TargetElemPtrPtr =
3883 Builder.CreateInBoundsGEP(Ty: RedListArrayTy, Ptr: ReduceList,
3884 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0),
3885 ConstantInt::get(Ty: IndexTy, V: En.index())});
3886 Value *TargetElemPtrVal =
3887 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: TargetElemPtrPtr);
3888 Value *TargetElemPtr = TargetElemPtrVal;
3889
3890 if (IsByRefElem && RI.DataPtrPtrGen) {
3891 InsertPointOrErrorTy GenRes =
3892 RI.DataPtrPtrGen(Builder.saveIP(), TargetElemPtr, TargetElemPtr);
3893
3894 if (!GenRes)
3895 return GenRes.takeError();
3896
3897 TargetElemPtr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: TargetElemPtr);
3898 }
3899
3900 if (NumIters > 1)
3901 TargetElemPtr =
3902 Builder.CreateGEP(Ty: Builder.getInt32Ty(), Ptr: TargetElemPtr, IdxList: Cnt);
3903
3904 // *TargetElemPtr = SrcMediumVal;
3905 Value *SrcMediumValue =
3906 Builder.CreateLoad(Ty: CType, Ptr: SrcMediumPtrVal, /*IsVolatile*/ isVolatile: true);
3907 Builder.CreateStore(Val: SrcMediumValue, Ptr: TargetElemPtr);
3908 Builder.CreateBr(Dest: W0MergeBB);
3909
3910 emitBlock(BB: W0ElseBB, CurFn: Builder.GetInsertBlock()->getParent());
3911 Builder.CreateBr(Dest: W0MergeBB);
3912
3913 emitBlock(BB: W0MergeBB, CurFn: Builder.GetInsertBlock()->getParent());
3914
3915 if (NumIters > 1) {
3916 Cnt = Builder.CreateNSWAdd(
3917 LHS: Cnt, RHS: ConstantInt::get(Ty: Builder.getInt32Ty(), /*V=*/1));
3918 Builder.CreateStore(Val: Cnt, Ptr: CntAddr, /*Volatile=*/isVolatile: false);
3919
3920 auto *CurFn = Builder.GetInsertBlock()->getParent();
3921 emitBranch(Target: PrecondBB);
3922 emitBlock(BB: ExitBB, CurFn);
3923 }
3924 RealTySize %= TySize;
3925 }
3926 }
3927
3928 Builder.CreateRetVoid();
3929
3930 return WcFunc;
3931}
3932
3933Expected<Function *> OpenMPIRBuilder::emitShuffleAndReduceFunction(
3934 ArrayRef<ReductionInfo> ReductionInfos, Function *ReduceFn,
3935 AttributeList FuncAttrs, ArrayRef<bool> IsByRef) {
3936 LLVMContext &Ctx = M.getContext();
3937 IRBuilder<>::InsertPointGuard IPG(Builder);
3938 FunctionType *FuncTy =
3939 FunctionType::get(Result: Builder.getVoidTy(),
3940 Params: {Builder.getPtrTy(), Builder.getInt16Ty(),
3941 Builder.getInt16Ty(), Builder.getInt16Ty()},
3942 /* IsVarArg */ isVarArg: false);
3943 Function *SarFunc =
3944 Function::Create(Ty: FuncTy, Linkage: GlobalVariable::InternalLinkage,
3945 N: "_omp_reduction_shuffle_and_reduce_func", M: &M);
3946 SarFunc->setCallingConv(Config.getRuntimeCC());
3947 SarFunc->setAttributes(FuncAttrs);
3948 SarFunc->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
3949 SarFunc->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
3950 SarFunc->addParamAttr(ArgNo: 2, Kind: Attribute::NoUndef);
3951 SarFunc->addParamAttr(ArgNo: 3, Kind: Attribute::NoUndef);
3952 SarFunc->addParamAttr(ArgNo: 1, Kind: Attribute::SExt);
3953 SarFunc->addParamAttr(ArgNo: 2, Kind: Attribute::SExt);
3954 SarFunc->addParamAttr(ArgNo: 3, Kind: Attribute::SExt);
3955 BasicBlock *EntryBB = BasicBlock::Create(Context&: M.getContext(), Name: "entry", Parent: SarFunc);
3956 Builder.SetInsertPoint(EntryBB);
3957 Builder.SetCurrentDebugLocation(llvm::DebugLoc());
3958
3959 // Thread local Reduce list used to host the values of data to be reduced.
3960 Argument *ReduceListArg = SarFunc->getArg(i: 0);
3961 // Current lane id; could be logical.
3962 Argument *LaneIDArg = SarFunc->getArg(i: 1);
3963 // Offset of the remote source lane relative to the current lane.
3964 Argument *RemoteLaneOffsetArg = SarFunc->getArg(i: 2);
3965 // Algorithm version. This is expected to be known at compile time.
3966 Argument *AlgoVerArg = SarFunc->getArg(i: 3);
3967
3968 Type *ReduceListArgType = ReduceListArg->getType();
3969 Type *LaneIDArgType = LaneIDArg->getType();
3970 Type *LaneIDArgPtrType = Builder.getPtrTy(AddrSpace: 0);
3971 Value *ReduceListAlloca = Builder.CreateAlloca(
3972 Ty: ReduceListArgType, ArraySize: nullptr, Name: ReduceListArg->getName() + ".addr");
3973 Value *LaneIdAlloca = Builder.CreateAlloca(Ty: LaneIDArgType, ArraySize: nullptr,
3974 Name: LaneIDArg->getName() + ".addr");
3975 Value *RemoteLaneOffsetAlloca = Builder.CreateAlloca(
3976 Ty: LaneIDArgType, ArraySize: nullptr, Name: RemoteLaneOffsetArg->getName() + ".addr");
3977 Value *AlgoVerAlloca = Builder.CreateAlloca(Ty: LaneIDArgType, ArraySize: nullptr,
3978 Name: AlgoVerArg->getName() + ".addr");
3979 ArrayType *RedListArrayTy =
3980 ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: ReductionInfos.size());
3981
3982 // Create a local thread-private variable to host the Reduce list
3983 // from a remote lane.
3984 Instruction *RemoteReductionListAlloca = Builder.CreateAlloca(
3985 Ty: RedListArrayTy, ArraySize: nullptr, Name: ".omp.reduction.remote_reduce_list");
3986
3987 Value *ReduceListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
3988 V: ReduceListAlloca, DestTy: ReduceListArgType,
3989 Name: ReduceListAlloca->getName() + ".ascast");
3990 Value *LaneIdAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
3991 V: LaneIdAlloca, DestTy: LaneIDArgPtrType, Name: LaneIdAlloca->getName() + ".ascast");
3992 Value *RemoteLaneOffsetAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
3993 V: RemoteLaneOffsetAlloca, DestTy: LaneIDArgPtrType,
3994 Name: RemoteLaneOffsetAlloca->getName() + ".ascast");
3995 Value *AlgoVerAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
3996 V: AlgoVerAlloca, DestTy: LaneIDArgPtrType, Name: AlgoVerAlloca->getName() + ".ascast");
3997 Value *RemoteListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
3998 V: RemoteReductionListAlloca, DestTy: Builder.getPtrTy(),
3999 Name: RemoteReductionListAlloca->getName() + ".ascast");
4000
4001 Builder.CreateStore(Val: ReduceListArg, Ptr: ReduceListAddrCast);
4002 Builder.CreateStore(Val: LaneIDArg, Ptr: LaneIdAddrCast);
4003 Builder.CreateStore(Val: RemoteLaneOffsetArg, Ptr: RemoteLaneOffsetAddrCast);
4004 Builder.CreateStore(Val: AlgoVerArg, Ptr: AlgoVerAddrCast);
4005
4006 Value *ReduceList = Builder.CreateLoad(Ty: ReduceListArgType, Ptr: ReduceListAddrCast);
4007 Value *LaneId = Builder.CreateLoad(Ty: LaneIDArgType, Ptr: LaneIdAddrCast);
4008 Value *RemoteLaneOffset =
4009 Builder.CreateLoad(Ty: LaneIDArgType, Ptr: RemoteLaneOffsetAddrCast);
4010 Value *AlgoVer = Builder.CreateLoad(Ty: LaneIDArgType, Ptr: AlgoVerAddrCast);
4011
4012 InsertPointTy AllocaIP = getInsertPointAfterInstr(I: RemoteReductionListAlloca);
4013
4014 // This loop iterates through the list of reduce elements and copies,
4015 // element by element, from a remote lane in the warp to RemoteReduceList,
4016 // hosted on the thread's stack.
4017 Error EmitRedLsCpRes = emitReductionListCopy(
4018 AllocaIP, Action: CopyAction::RemoteLaneToThread, ReductionArrayTy: RedListArrayTy, ReductionInfos,
4019 SrcBase: ReduceList, DestBase: RemoteListAddrCast, IsByRef,
4020 CopyOptions: {.RemoteLaneOffset: RemoteLaneOffset, .ScratchpadIndex: nullptr, .ScratchpadWidth: nullptr});
4021
4022 if (EmitRedLsCpRes)
4023 return EmitRedLsCpRes;
4024
4025 // The actions to be performed on the Remote Reduce list is dependent
4026 // on the algorithm version.
4027 //
4028 // if (AlgoVer==0) || (AlgoVer==1 && (LaneId < Offset)) || (AlgoVer==2 &&
4029 // LaneId % 2 == 0 && Offset > 0):
4030 // do the reduction value aggregation
4031 //
4032 // The thread local variable Reduce list is mutated in place to host the
4033 // reduced data, which is the aggregated value produced from local and
4034 // remote lanes.
4035 //
4036 // Note that AlgoVer is expected to be a constant integer known at compile
4037 // time.
4038 // When AlgoVer==0, the first conjunction evaluates to true, making
4039 // the entire predicate true during compile time.
4040 // When AlgoVer==1, the second conjunction has only the second part to be
4041 // evaluated during runtime. Other conjunctions evaluates to false
4042 // during compile time.
4043 // When AlgoVer==2, the third conjunction has only the second part to be
4044 // evaluated during runtime. Other conjunctions evaluates to false
4045 // during compile time.
4046 Value *CondAlgo0 = Builder.CreateIsNull(Arg: AlgoVer);
4047 Value *Algo1 = Builder.CreateICmpEQ(LHS: AlgoVer, RHS: Builder.getInt16(C: 1));
4048 Value *LaneComp = Builder.CreateICmpULT(LHS: LaneId, RHS: RemoteLaneOffset);
4049 Value *CondAlgo1 = Builder.CreateAnd(LHS: Algo1, RHS: LaneComp);
4050 Value *Algo2 = Builder.CreateICmpEQ(LHS: AlgoVer, RHS: Builder.getInt16(C: 2));
4051 Value *LaneIdAnd1 = Builder.CreateAnd(LHS: LaneId, RHS: Builder.getInt16(C: 1));
4052 Value *LaneIdComp = Builder.CreateIsNull(Arg: LaneIdAnd1);
4053 Value *Algo2AndLaneIdComp = Builder.CreateAnd(LHS: Algo2, RHS: LaneIdComp);
4054 Value *RemoteOffsetComp =
4055 Builder.CreateICmpSGT(LHS: RemoteLaneOffset, RHS: Builder.getInt16(C: 0));
4056 Value *CondAlgo2 = Builder.CreateAnd(LHS: Algo2AndLaneIdComp, RHS: RemoteOffsetComp);
4057 Value *CA0OrCA1 = Builder.CreateOr(LHS: CondAlgo0, RHS: CondAlgo1);
4058 Value *CondReduce = Builder.CreateOr(LHS: CA0OrCA1, RHS: CondAlgo2);
4059
4060 BasicBlock *ThenBB = BasicBlock::Create(Context&: Ctx, Name: "then");
4061 BasicBlock *ElseBB = BasicBlock::Create(Context&: Ctx, Name: "else");
4062 BasicBlock *MergeBB = BasicBlock::Create(Context&: Ctx, Name: "ifcont");
4063
4064 Builder.CreateCondBr(Cond: CondReduce, True: ThenBB, False: ElseBB);
4065 emitBlock(BB: ThenBB, CurFn: Builder.GetInsertBlock()->getParent());
4066 Value *LocalReduceListPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
4067 V: ReduceList, DestTy: Builder.getPtrTy());
4068 Value *RemoteReduceListPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
4069 V: RemoteListAddrCast, DestTy: Builder.getPtrTy());
4070 createRuntimeFunctionCall(Callee: ReduceFn, Args: {LocalReduceListPtr, RemoteReduceListPtr})
4071 ->addFnAttr(Kind: Attribute::NoUnwind);
4072 Builder.CreateBr(Dest: MergeBB);
4073
4074 emitBlock(BB: ElseBB, CurFn: Builder.GetInsertBlock()->getParent());
4075 Builder.CreateBr(Dest: MergeBB);
4076
4077 emitBlock(BB: MergeBB, CurFn: Builder.GetInsertBlock()->getParent());
4078
4079 // if (AlgoVer==1 && (LaneId >= Offset)) copy Remote Reduce list to local
4080 // Reduce list.
4081 Algo1 = Builder.CreateICmpEQ(LHS: AlgoVer, RHS: Builder.getInt16(C: 1));
4082 Value *LaneIdGtOffset = Builder.CreateICmpUGE(LHS: LaneId, RHS: RemoteLaneOffset);
4083 Value *CondCopy = Builder.CreateAnd(LHS: Algo1, RHS: LaneIdGtOffset);
4084
4085 BasicBlock *CpyThenBB = BasicBlock::Create(Context&: Ctx, Name: "then");
4086 BasicBlock *CpyElseBB = BasicBlock::Create(Context&: Ctx, Name: "else");
4087 BasicBlock *CpyMergeBB = BasicBlock::Create(Context&: Ctx, Name: "ifcont");
4088 Builder.CreateCondBr(Cond: CondCopy, True: CpyThenBB, False: CpyElseBB);
4089
4090 emitBlock(BB: CpyThenBB, CurFn: Builder.GetInsertBlock()->getParent());
4091
4092 EmitRedLsCpRes = emitReductionListCopy(
4093 AllocaIP, Action: CopyAction::ThreadCopy, ReductionArrayTy: RedListArrayTy, ReductionInfos,
4094 SrcBase: RemoteListAddrCast, DestBase: ReduceList, IsByRef);
4095
4096 if (EmitRedLsCpRes)
4097 return EmitRedLsCpRes;
4098
4099 Builder.CreateBr(Dest: CpyMergeBB);
4100
4101 emitBlock(BB: CpyElseBB, CurFn: Builder.GetInsertBlock()->getParent());
4102 Builder.CreateBr(Dest: CpyMergeBB);
4103
4104 emitBlock(BB: CpyMergeBB, CurFn: Builder.GetInsertBlock()->getParent());
4105
4106 Builder.CreateRetVoid();
4107
4108 return SarFunc;
4109}
4110
4111OpenMPIRBuilder::InsertPointOrErrorTy
4112OpenMPIRBuilder::generateReductionDescriptor(
4113 Value *DescriptorAddr, Value *DataPtr, Value *SrcDescriptorAddr,
4114 Type *DescriptorType,
4115 function_ref<InsertPointOrErrorTy(InsertPointTy, Value *, Value *&)>
4116 DataPtrPtrGen) {
4117
4118 // Copy the source descriptor to preserve all metadata (rank, extents,
4119 // strides, etc.)
4120 Value *DescriptorSize =
4121 Builder.getInt64(C: M.getDataLayout().getTypeStoreSize(Ty: DescriptorType));
4122 Builder.CreateMemCpy(
4123 Dst: DescriptorAddr, DstAlign: M.getDataLayout().getPrefTypeAlign(Ty: DescriptorType),
4124 Src: SrcDescriptorAddr, SrcAlign: M.getDataLayout().getPrefTypeAlign(Ty: DescriptorType),
4125 Size: DescriptorSize);
4126
4127 // Update the base pointer field to point to the local shuffled data
4128 Value *DataPtrField;
4129 InsertPointOrErrorTy GenResult =
4130 DataPtrPtrGen(Builder.saveIP(), DescriptorAddr, DataPtrField);
4131
4132 if (!GenResult)
4133 return GenResult.takeError();
4134
4135 Builder.CreateStore(Val: Builder.CreatePointerBitCastOrAddrSpaceCast(
4136 V: DataPtr, DestTy: Builder.getPtrTy(), Name: ".ascast"),
4137 Ptr: DataPtrField);
4138
4139 return Builder.saveIP();
4140}
4141
4142Expected<Value *> OpenMPIRBuilder::createReductionDescriptorCopy(
4143 InsertPointTy AllocaIP, const ReductionInfo &RI, Value *DataPtr,
4144 Value *SrcDescriptorAddr, Type *DescriptorPtrTy, const Twine &Name) {
4145 InsertPointTy OldIP = Builder.saveIP();
4146 Builder.restoreIP(IP: AllocaIP);
4147
4148 AllocaInst *DescriptorAlloca =
4149 Builder.CreateAlloca(Ty: RI.ByRefAllocatedType, ArraySize: nullptr, Name);
4150 DescriptorAlloca->setAlignment(
4151 M.getDataLayout().getPrefTypeAlign(Ty: RI.ByRefAllocatedType));
4152 Value *DescriptorAddr = Builder.CreatePointerBitCastOrAddrSpaceCast(
4153 V: DescriptorAlloca, DestTy: DescriptorPtrTy,
4154 Name: DescriptorAlloca->getName() + ".ascast");
4155
4156 Builder.restoreIP(IP: OldIP);
4157
4158 InsertPointOrErrorTy GenResult =
4159 generateReductionDescriptor(DescriptorAddr, DataPtr, SrcDescriptorAddr,
4160 DescriptorType: RI.ByRefAllocatedType, DataPtrPtrGen: RI.DataPtrPtrGen);
4161 if (!GenResult)
4162 return GenResult.takeError();
4163
4164 return DescriptorAddr;
4165}
4166
4167Expected<Function *> OpenMPIRBuilder::emitListToGlobalCopyFunction(
4168 ArrayRef<ReductionInfo> ReductionInfos, Type *ReductionsBufferTy,
4169 AttributeList FuncAttrs, ArrayRef<bool> IsByRef) {
4170 IRBuilder<>::InsertPointGuard IPG(Builder);
4171 LLVMContext &Ctx = M.getContext();
4172 FunctionType *FuncTy = FunctionType::get(
4173 Result: Builder.getVoidTy(),
4174 Params: {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},
4175 /* IsVarArg */ isVarArg: false);
4176 Function *LtGCFunc =
4177 Function::Create(Ty: FuncTy, Linkage: GlobalVariable::InternalLinkage,
4178 N: "_omp_reduction_list_to_global_copy_func", M: &M);
4179 LtGCFunc->setAttributes(FuncAttrs);
4180 LtGCFunc->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
4181 LtGCFunc->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
4182 LtGCFunc->addParamAttr(ArgNo: 2, Kind: Attribute::NoUndef);
4183
4184 BasicBlock *EntryBlock = BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: LtGCFunc);
4185 Builder.SetInsertPoint(EntryBlock);
4186 Builder.SetCurrentDebugLocation(llvm::DebugLoc());
4187
4188 // Buffer: global reduction buffer.
4189 Argument *BufferArg = LtGCFunc->getArg(i: 0);
4190 // Idx: index of the buffer.
4191 Argument *IdxArg = LtGCFunc->getArg(i: 1);
4192 // ReduceList: thread local Reduce list.
4193 Argument *ReduceListArg = LtGCFunc->getArg(i: 2);
4194
4195 Value *BufferArgAlloca = Builder.CreateAlloca(Ty: Builder.getPtrTy(), ArraySize: nullptr,
4196 Name: BufferArg->getName() + ".addr");
4197 Value *IdxArgAlloca = Builder.CreateAlloca(Ty: Builder.getInt32Ty(), ArraySize: nullptr,
4198 Name: IdxArg->getName() + ".addr");
4199 Value *ReduceListArgAlloca = Builder.CreateAlloca(
4200 Ty: Builder.getPtrTy(), ArraySize: nullptr, Name: ReduceListArg->getName() + ".addr");
4201 Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4202 V: BufferArgAlloca, DestTy: Builder.getPtrTy(),
4203 Name: BufferArgAlloca->getName() + ".ascast");
4204 Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4205 V: IdxArgAlloca, DestTy: Builder.getPtrTy(), Name: IdxArgAlloca->getName() + ".ascast");
4206 Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4207 V: ReduceListArgAlloca, DestTy: Builder.getPtrTy(),
4208 Name: ReduceListArgAlloca->getName() + ".ascast");
4209
4210 Builder.CreateStore(Val: BufferArg, Ptr: BufferArgAddrCast);
4211 Builder.CreateStore(Val: IdxArg, Ptr: IdxArgAddrCast);
4212 Builder.CreateStore(Val: ReduceListArg, Ptr: ReduceListArgAddrCast);
4213
4214 Value *LocalReduceList =
4215 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ReduceListArgAddrCast);
4216 Value *BufferArgVal =
4217 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: BufferArgAddrCast);
4218 Value *Idxs[] = {Builder.CreateLoad(Ty: Builder.getInt32Ty(), Ptr: IdxArgAddrCast)};
4219 Type *IndexTy = Builder.getIndexTy(
4220 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
4221 for (auto En : enumerate(First&: ReductionInfos)) {
4222 const ReductionInfo &RI = En.value();
4223 auto *RedListArrayTy =
4224 ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: ReductionInfos.size());
4225 // Reduce element = LocalReduceList[i]
4226 Value *ElemPtrPtr = Builder.CreateInBoundsGEP(
4227 Ty: RedListArrayTy, Ptr: LocalReduceList,
4228 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
4229 // elemptr = ((CopyType*)(elemptrptr)) + I
4230 Value *ElemPtr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ElemPtrPtr);
4231
4232 // Global = Buffer.VD[Idx];
4233 Value *BufferVD =
4234 Builder.CreateInBoundsGEP(Ty: ReductionsBufferTy, Ptr: BufferArgVal, IdxList: Idxs);
4235 Value *GlobVal = Builder.CreateConstInBoundsGEP2_32(
4236 Ty: ReductionsBufferTy, Ptr: BufferVD, Idx0: 0, Idx1: En.index());
4237
4238 switch (RI.EvaluationKind) {
4239 case EvalKind::Scalar: {
4240 Value *TargetElement;
4241
4242 if (IsByRef.empty() || !IsByRef[En.index()]) {
4243 TargetElement = Builder.CreateLoad(Ty: RI.ElementType, Ptr: ElemPtr);
4244 } else {
4245 if (RI.DataPtrPtrGen) {
4246 InsertPointOrErrorTy GenResult =
4247 RI.DataPtrPtrGen(Builder.saveIP(), ElemPtr, ElemPtr);
4248
4249 if (!GenResult)
4250 return GenResult.takeError();
4251
4252 ElemPtr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ElemPtr);
4253 }
4254 TargetElement = Builder.CreateLoad(Ty: RI.ByRefElementType, Ptr: ElemPtr);
4255 }
4256
4257 Builder.CreateStore(Val: TargetElement, Ptr: GlobVal);
4258 break;
4259 }
4260 case EvalKind::Complex: {
4261 Value *SrcRealPtr = Builder.CreateConstInBoundsGEP2_32(
4262 Ty: RI.ElementType, Ptr: ElemPtr, Idx0: 0, Idx1: 0, Name: ".realp");
4263 Value *SrcReal = Builder.CreateLoad(
4264 Ty: RI.ElementType->getStructElementType(N: 0), Ptr: SrcRealPtr, Name: ".real");
4265 Value *SrcImgPtr = Builder.CreateConstInBoundsGEP2_32(
4266 Ty: RI.ElementType, Ptr: ElemPtr, Idx0: 0, Idx1: 1, Name: ".imagp");
4267 Value *SrcImg = Builder.CreateLoad(
4268 Ty: RI.ElementType->getStructElementType(N: 1), Ptr: SrcImgPtr, Name: ".imag");
4269
4270 Value *DestRealPtr = Builder.CreateConstInBoundsGEP2_32(
4271 Ty: RI.ElementType, Ptr: GlobVal, Idx0: 0, Idx1: 0, Name: ".realp");
4272 Value *DestImgPtr = Builder.CreateConstInBoundsGEP2_32(
4273 Ty: RI.ElementType, Ptr: GlobVal, Idx0: 0, Idx1: 1, Name: ".imagp");
4274 Builder.CreateStore(Val: SrcReal, Ptr: DestRealPtr);
4275 Builder.CreateStore(Val: SrcImg, Ptr: DestImgPtr);
4276 break;
4277 }
4278 case EvalKind::Aggregate: {
4279 Value *SizeVal =
4280 Builder.getInt64(C: M.getDataLayout().getTypeStoreSize(Ty: RI.ElementType));
4281 Builder.CreateMemCpy(
4282 Dst: GlobVal, DstAlign: M.getDataLayout().getPrefTypeAlign(Ty: RI.ElementType), Src: ElemPtr,
4283 SrcAlign: M.getDataLayout().getPrefTypeAlign(Ty: RI.ElementType), Size: SizeVal, isVolatile: false);
4284 break;
4285 }
4286 }
4287 }
4288
4289 Builder.CreateRetVoid();
4290 return LtGCFunc;
4291}
4292
4293Expected<Function *> OpenMPIRBuilder::emitListToGlobalReduceFunction(
4294 ArrayRef<ReductionInfo> ReductionInfos, Function *ReduceFn,
4295 Type *ReductionsBufferTy, AttributeList FuncAttrs, ArrayRef<bool> IsByRef) {
4296 IRBuilder<>::InsertPointGuard IPG(Builder);
4297 LLVMContext &Ctx = M.getContext();
4298 FunctionType *FuncTy = FunctionType::get(
4299 Result: Builder.getVoidTy(),
4300 Params: {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},
4301 /* IsVarArg */ isVarArg: false);
4302 Function *LtGRFunc =
4303 Function::Create(Ty: FuncTy, Linkage: GlobalVariable::InternalLinkage,
4304 N: "_omp_reduction_list_to_global_reduce_func", M: &M);
4305 LtGRFunc->setAttributes(FuncAttrs);
4306 LtGRFunc->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
4307 LtGRFunc->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
4308 LtGRFunc->addParamAttr(ArgNo: 2, Kind: Attribute::NoUndef);
4309
4310 BasicBlock *EntryBlock = BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: LtGRFunc);
4311 Builder.SetInsertPoint(EntryBlock);
4312 Builder.SetCurrentDebugLocation(llvm::DebugLoc());
4313
4314 // Buffer: global reduction buffer.
4315 Argument *BufferArg = LtGRFunc->getArg(i: 0);
4316 // Idx: index of the buffer.
4317 Argument *IdxArg = LtGRFunc->getArg(i: 1);
4318 // ReduceList: thread local Reduce list.
4319 Argument *ReduceListArg = LtGRFunc->getArg(i: 2);
4320
4321 Value *BufferArgAlloca = Builder.CreateAlloca(Ty: Builder.getPtrTy(), ArraySize: nullptr,
4322 Name: BufferArg->getName() + ".addr");
4323 Value *IdxArgAlloca = Builder.CreateAlloca(Ty: Builder.getInt32Ty(), ArraySize: nullptr,
4324 Name: IdxArg->getName() + ".addr");
4325 Value *ReduceListArgAlloca = Builder.CreateAlloca(
4326 Ty: Builder.getPtrTy(), ArraySize: nullptr, Name: ReduceListArg->getName() + ".addr");
4327 auto *RedListArrayTy =
4328 ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: ReductionInfos.size());
4329
4330 // 1. Build a list of reduction variables.
4331 // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
4332 Value *LocalReduceList =
4333 Builder.CreateAlloca(Ty: RedListArrayTy, ArraySize: nullptr, Name: ".omp.reduction.red_list");
4334
4335 InsertPointTy AllocaIP{EntryBlock, EntryBlock->begin()};
4336
4337 Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4338 V: BufferArgAlloca, DestTy: Builder.getPtrTy(),
4339 Name: BufferArgAlloca->getName() + ".ascast");
4340 Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4341 V: IdxArgAlloca, DestTy: Builder.getPtrTy(), Name: IdxArgAlloca->getName() + ".ascast");
4342 Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4343 V: ReduceListArgAlloca, DestTy: Builder.getPtrTy(),
4344 Name: ReduceListArgAlloca->getName() + ".ascast");
4345 Value *LocalReduceListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4346 V: LocalReduceList, DestTy: Builder.getPtrTy(),
4347 Name: LocalReduceList->getName() + ".ascast");
4348
4349 Builder.CreateStore(Val: BufferArg, Ptr: BufferArgAddrCast);
4350 Builder.CreateStore(Val: IdxArg, Ptr: IdxArgAddrCast);
4351 Builder.CreateStore(Val: ReduceListArg, Ptr: ReduceListArgAddrCast);
4352
4353 Value *BufferVal = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: BufferArgAddrCast);
4354 Value *Idxs[] = {Builder.CreateLoad(Ty: Builder.getInt32Ty(), Ptr: IdxArgAddrCast)};
4355 Type *IndexTy = Builder.getIndexTy(
4356 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
4357 for (auto En : enumerate(First&: ReductionInfos)) {
4358 const ReductionInfo &RI = En.value();
4359
4360 Value *TargetElementPtrPtr = Builder.CreateInBoundsGEP(
4361 Ty: RedListArrayTy, Ptr: LocalReduceListAddrCast,
4362 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
4363 Value *BufferVD =
4364 Builder.CreateInBoundsGEP(Ty: ReductionsBufferTy, Ptr: BufferVal, IdxList: Idxs);
4365 // Global = Buffer.VD[Idx];
4366 Value *GlobValPtr = Builder.CreateConstInBoundsGEP2_32(
4367 Ty: ReductionsBufferTy, Ptr: BufferVD, Idx0: 0, Idx1: En.index());
4368
4369 if (!IsByRef.empty() && IsByRef[En.index()] && RI.DataPtrPtrGen) {
4370 // Get source descriptor from the reduce list argument
4371 Value *ReduceList =
4372 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ReduceListArgAddrCast);
4373 Value *SrcElementPtrPtr =
4374 Builder.CreateInBoundsGEP(Ty: RedListArrayTy, Ptr: ReduceList,
4375 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0),
4376 ConstantInt::get(Ty: IndexTy, V: En.index())});
4377 Value *SrcDescriptorAddr =
4378 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: SrcElementPtrPtr);
4379
4380 // Copy descriptor from source and update base_ptr to global buffer data
4381 Expected<Value *> ByRefAlloc = createReductionDescriptorCopy(
4382 AllocaIP, RI, DataPtr: GlobValPtr, SrcDescriptorAddr, DescriptorPtrTy: Builder.getPtrTy());
4383 if (!ByRefAlloc)
4384 return ByRefAlloc.takeError();
4385
4386 Builder.CreateStore(Val: *ByRefAlloc, Ptr: TargetElementPtrPtr);
4387 } else {
4388 Builder.CreateStore(Val: GlobValPtr, Ptr: TargetElementPtrPtr);
4389 }
4390 }
4391
4392 // Call reduce_function(GlobalReduceList, ReduceList)
4393 Value *ReduceList =
4394 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ReduceListArgAddrCast);
4395 createRuntimeFunctionCall(Callee: ReduceFn, Args: {LocalReduceListAddrCast, ReduceList})
4396 ->addFnAttr(Kind: Attribute::NoUnwind);
4397 Builder.CreateRetVoid();
4398 return LtGRFunc;
4399}
4400
4401Expected<Function *> OpenMPIRBuilder::emitGlobalToListCopyFunction(
4402 ArrayRef<ReductionInfo> ReductionInfos, Type *ReductionsBufferTy,
4403 AttributeList FuncAttrs, ArrayRef<bool> IsByRef) {
4404 IRBuilder<>::InsertPointGuard IPG(Builder);
4405 LLVMContext &Ctx = M.getContext();
4406 FunctionType *FuncTy = FunctionType::get(
4407 Result: Builder.getVoidTy(),
4408 Params: {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},
4409 /* IsVarArg */ isVarArg: false);
4410 Function *GtLCFunc =
4411 Function::Create(Ty: FuncTy, Linkage: GlobalVariable::InternalLinkage,
4412 N: "_omp_reduction_global_to_list_copy_func", M: &M);
4413 GtLCFunc->setAttributes(FuncAttrs);
4414 GtLCFunc->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
4415 GtLCFunc->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
4416 GtLCFunc->addParamAttr(ArgNo: 2, Kind: Attribute::NoUndef);
4417
4418 BasicBlock *EntryBlock = BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: GtLCFunc);
4419 Builder.SetInsertPoint(EntryBlock);
4420 Builder.SetCurrentDebugLocation(llvm::DebugLoc());
4421
4422 // Buffer: global reduction buffer.
4423 Argument *BufferArg = GtLCFunc->getArg(i: 0);
4424 // Idx: index of the buffer.
4425 Argument *IdxArg = GtLCFunc->getArg(i: 1);
4426 // ReduceList: thread local Reduce list.
4427 Argument *ReduceListArg = GtLCFunc->getArg(i: 2);
4428
4429 Value *BufferArgAlloca = Builder.CreateAlloca(Ty: Builder.getPtrTy(), ArraySize: nullptr,
4430 Name: BufferArg->getName() + ".addr");
4431 Value *IdxArgAlloca = Builder.CreateAlloca(Ty: Builder.getInt32Ty(), ArraySize: nullptr,
4432 Name: IdxArg->getName() + ".addr");
4433 Value *ReduceListArgAlloca = Builder.CreateAlloca(
4434 Ty: Builder.getPtrTy(), ArraySize: nullptr, Name: ReduceListArg->getName() + ".addr");
4435 Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4436 V: BufferArgAlloca, DestTy: Builder.getPtrTy(),
4437 Name: BufferArgAlloca->getName() + ".ascast");
4438 Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4439 V: IdxArgAlloca, DestTy: Builder.getPtrTy(), Name: IdxArgAlloca->getName() + ".ascast");
4440 Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4441 V: ReduceListArgAlloca, DestTy: Builder.getPtrTy(),
4442 Name: ReduceListArgAlloca->getName() + ".ascast");
4443 Builder.CreateStore(Val: BufferArg, Ptr: BufferArgAddrCast);
4444 Builder.CreateStore(Val: IdxArg, Ptr: IdxArgAddrCast);
4445 Builder.CreateStore(Val: ReduceListArg, Ptr: ReduceListArgAddrCast);
4446
4447 Value *LocalReduceList =
4448 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ReduceListArgAddrCast);
4449 Value *BufferVal = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: BufferArgAddrCast);
4450 Value *Idxs[] = {Builder.CreateLoad(Ty: Builder.getInt32Ty(), Ptr: IdxArgAddrCast)};
4451 Type *IndexTy = Builder.getIndexTy(
4452 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
4453 for (auto En : enumerate(First&: ReductionInfos)) {
4454 const OpenMPIRBuilder::ReductionInfo &RI = En.value();
4455 auto *RedListArrayTy =
4456 ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: ReductionInfos.size());
4457 // Reduce element = LocalReduceList[i]
4458 Value *ElemPtrPtr = Builder.CreateInBoundsGEP(
4459 Ty: RedListArrayTy, Ptr: LocalReduceList,
4460 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
4461 // elemptr = ((CopyType*)(elemptrptr)) + I
4462 Value *ElemPtr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ElemPtrPtr);
4463 // Global = Buffer.VD[Idx];
4464 Value *BufferVD =
4465 Builder.CreateInBoundsGEP(Ty: ReductionsBufferTy, Ptr: BufferVal, IdxList: Idxs);
4466 Value *GlobValPtr = Builder.CreateConstInBoundsGEP2_32(
4467 Ty: ReductionsBufferTy, Ptr: BufferVD, Idx0: 0, Idx1: En.index());
4468
4469 switch (RI.EvaluationKind) {
4470 case EvalKind::Scalar: {
4471 Type *ElemType = RI.ElementType;
4472
4473 if (!IsByRef.empty() && IsByRef[En.index()]) {
4474 ElemType = RI.ByRefElementType;
4475 if (RI.DataPtrPtrGen) {
4476 InsertPointOrErrorTy GenResult =
4477 RI.DataPtrPtrGen(Builder.saveIP(), ElemPtr, ElemPtr);
4478
4479 if (!GenResult)
4480 return GenResult.takeError();
4481
4482 ElemPtr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ElemPtr);
4483 }
4484 }
4485
4486 Value *TargetElement = Builder.CreateLoad(Ty: ElemType, Ptr: GlobValPtr);
4487 Builder.CreateStore(Val: TargetElement, Ptr: ElemPtr);
4488 break;
4489 }
4490 case EvalKind::Complex: {
4491 Value *SrcRealPtr = Builder.CreateConstInBoundsGEP2_32(
4492 Ty: RI.ElementType, Ptr: GlobValPtr, Idx0: 0, Idx1: 0, Name: ".realp");
4493 Value *SrcReal = Builder.CreateLoad(
4494 Ty: RI.ElementType->getStructElementType(N: 0), Ptr: SrcRealPtr, Name: ".real");
4495 Value *SrcImgPtr = Builder.CreateConstInBoundsGEP2_32(
4496 Ty: RI.ElementType, Ptr: GlobValPtr, Idx0: 0, Idx1: 1, Name: ".imagp");
4497 Value *SrcImg = Builder.CreateLoad(
4498 Ty: RI.ElementType->getStructElementType(N: 1), Ptr: SrcImgPtr, Name: ".imag");
4499
4500 Value *DestRealPtr = Builder.CreateConstInBoundsGEP2_32(
4501 Ty: RI.ElementType, Ptr: ElemPtr, Idx0: 0, Idx1: 0, Name: ".realp");
4502 Value *DestImgPtr = Builder.CreateConstInBoundsGEP2_32(
4503 Ty: RI.ElementType, Ptr: ElemPtr, Idx0: 0, Idx1: 1, Name: ".imagp");
4504 Builder.CreateStore(Val: SrcReal, Ptr: DestRealPtr);
4505 Builder.CreateStore(Val: SrcImg, Ptr: DestImgPtr);
4506 break;
4507 }
4508 case EvalKind::Aggregate: {
4509 Value *SizeVal =
4510 Builder.getInt64(C: M.getDataLayout().getTypeStoreSize(Ty: RI.ElementType));
4511 Builder.CreateMemCpy(
4512 Dst: ElemPtr, DstAlign: M.getDataLayout().getPrefTypeAlign(Ty: RI.ElementType),
4513 Src: GlobValPtr, SrcAlign: M.getDataLayout().getPrefTypeAlign(Ty: RI.ElementType),
4514 Size: SizeVal, isVolatile: false);
4515 break;
4516 }
4517 }
4518 }
4519
4520 Builder.CreateRetVoid();
4521 return GtLCFunc;
4522}
4523
4524Expected<Function *> OpenMPIRBuilder::emitGlobalToListReduceFunction(
4525 ArrayRef<ReductionInfo> ReductionInfos, Function *ReduceFn,
4526 Type *ReductionsBufferTy, AttributeList FuncAttrs, ArrayRef<bool> IsByRef) {
4527 IRBuilder<>::InsertPointGuard IPG(Builder);
4528 LLVMContext &Ctx = M.getContext();
4529 auto *FuncTy = FunctionType::get(
4530 Result: Builder.getVoidTy(),
4531 Params: {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},
4532 /* IsVarArg */ isVarArg: false);
4533 Function *GtLRFunc =
4534 Function::Create(Ty: FuncTy, Linkage: GlobalVariable::InternalLinkage,
4535 N: "_omp_reduction_global_to_list_reduce_func", M: &M);
4536 GtLRFunc->setAttributes(FuncAttrs);
4537 GtLRFunc->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
4538 GtLRFunc->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
4539 GtLRFunc->addParamAttr(ArgNo: 2, Kind: Attribute::NoUndef);
4540
4541 BasicBlock *EntryBlock = BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: GtLRFunc);
4542 Builder.SetInsertPoint(EntryBlock);
4543 Builder.SetCurrentDebugLocation(llvm::DebugLoc());
4544
4545 // Buffer: global reduction buffer.
4546 Argument *BufferArg = GtLRFunc->getArg(i: 0);
4547 // Idx: index of the buffer.
4548 Argument *IdxArg = GtLRFunc->getArg(i: 1);
4549 // ReduceList: thread local Reduce list.
4550 Argument *ReduceListArg = GtLRFunc->getArg(i: 2);
4551
4552 Value *BufferArgAlloca = Builder.CreateAlloca(Ty: Builder.getPtrTy(), ArraySize: nullptr,
4553 Name: BufferArg->getName() + ".addr");
4554 Value *IdxArgAlloca = Builder.CreateAlloca(Ty: Builder.getInt32Ty(), ArraySize: nullptr,
4555 Name: IdxArg->getName() + ".addr");
4556 Value *ReduceListArgAlloca = Builder.CreateAlloca(
4557 Ty: Builder.getPtrTy(), ArraySize: nullptr, Name: ReduceListArg->getName() + ".addr");
4558 ArrayType *RedListArrayTy =
4559 ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: ReductionInfos.size());
4560
4561 // 1. Build a list of reduction variables.
4562 // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
4563 Value *LocalReduceList =
4564 Builder.CreateAlloca(Ty: RedListArrayTy, ArraySize: nullptr, Name: ".omp.reduction.red_list");
4565
4566 InsertPointTy AllocaIP{EntryBlock, EntryBlock->begin()};
4567
4568 Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4569 V: BufferArgAlloca, DestTy: Builder.getPtrTy(),
4570 Name: BufferArgAlloca->getName() + ".ascast");
4571 Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4572 V: IdxArgAlloca, DestTy: Builder.getPtrTy(), Name: IdxArgAlloca->getName() + ".ascast");
4573 Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4574 V: ReduceListArgAlloca, DestTy: Builder.getPtrTy(),
4575 Name: ReduceListArgAlloca->getName() + ".ascast");
4576 Value *ReductionList = Builder.CreatePointerBitCastOrAddrSpaceCast(
4577 V: LocalReduceList, DestTy: Builder.getPtrTy(),
4578 Name: LocalReduceList->getName() + ".ascast");
4579
4580 Builder.CreateStore(Val: BufferArg, Ptr: BufferArgAddrCast);
4581 Builder.CreateStore(Val: IdxArg, Ptr: IdxArgAddrCast);
4582 Builder.CreateStore(Val: ReduceListArg, Ptr: ReduceListArgAddrCast);
4583
4584 Value *BufferVal = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: BufferArgAddrCast);
4585 Value *Idxs[] = {Builder.CreateLoad(Ty: Builder.getInt32Ty(), Ptr: IdxArgAddrCast)};
4586 Type *IndexTy = Builder.getIndexTy(
4587 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
4588 for (auto En : enumerate(First&: ReductionInfos)) {
4589 const ReductionInfo &RI = En.value();
4590
4591 Value *TargetElementPtrPtr = Builder.CreateInBoundsGEP(
4592 Ty: RedListArrayTy, Ptr: ReductionList,
4593 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
4594 // Global = Buffer.VD[Idx];
4595 Value *BufferVD =
4596 Builder.CreateInBoundsGEP(Ty: ReductionsBufferTy, Ptr: BufferVal, IdxList: Idxs);
4597 Value *GlobValPtr = Builder.CreateConstInBoundsGEP2_32(
4598 Ty: ReductionsBufferTy, Ptr: BufferVD, Idx0: 0, Idx1: En.index());
4599
4600 if (!IsByRef.empty() && IsByRef[En.index()] && RI.DataPtrPtrGen) {
4601 // Get source descriptor from the reduce list
4602 Value *ReduceListVal =
4603 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ReduceListArgAddrCast);
4604 Value *SrcElementPtrPtr =
4605 Builder.CreateInBoundsGEP(Ty: RedListArrayTy, Ptr: ReduceListVal,
4606 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0),
4607 ConstantInt::get(Ty: IndexTy, V: En.index())});
4608 Value *SrcDescriptorAddr =
4609 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: SrcElementPtrPtr);
4610
4611 // Copy descriptor from source and update base_ptr to global buffer data
4612 Expected<Value *> ByRefAlloc = createReductionDescriptorCopy(
4613 AllocaIP, RI, DataPtr: GlobValPtr, SrcDescriptorAddr, DescriptorPtrTy: Builder.getPtrTy());
4614 if (!ByRefAlloc)
4615 return ByRefAlloc.takeError();
4616
4617 Builder.CreateStore(Val: *ByRefAlloc, Ptr: TargetElementPtrPtr);
4618 } else {
4619 Builder.CreateStore(Val: GlobValPtr, Ptr: TargetElementPtrPtr);
4620 }
4621 }
4622
4623 // Call reduce_function(ReduceList, GlobalReduceList)
4624 Value *ReduceList =
4625 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ReduceListArgAddrCast);
4626 createRuntimeFunctionCall(Callee: ReduceFn, Args: {ReduceList, ReductionList})
4627 ->addFnAttr(Kind: Attribute::NoUnwind);
4628 Builder.CreateRetVoid();
4629 return GtLRFunc;
4630}
4631
4632std::string OpenMPIRBuilder::getReductionFuncName(StringRef Name) const {
4633 std::string Suffix =
4634 createPlatformSpecificName(Parts: {"omp", "reduction", "reduction_func"});
4635 return (Name + Suffix).str();
4636}
4637
4638Expected<Function *> OpenMPIRBuilder::createReductionFunction(
4639 StringRef ReducerName, ArrayRef<ReductionInfo> ReductionInfos,
4640 ArrayRef<bool> IsByRef, ReductionGenCBKind ReductionGenCBKind,
4641 AttributeList FuncAttrs) {
4642 IRBuilder<>::InsertPointGuard IPG(Builder);
4643 auto *FuncTy = FunctionType::get(Result: Builder.getVoidTy(),
4644 Params: {Builder.getPtrTy(), Builder.getPtrTy()},
4645 /* IsVarArg */ isVarArg: false);
4646 std::string Name = getReductionFuncName(Name: ReducerName);
4647 Function *ReductionFunc =
4648 Function::Create(Ty: FuncTy, Linkage: GlobalVariable::InternalLinkage, N: Name, M: &M);
4649 ReductionFunc->setCallingConv(Config.getRuntimeCC());
4650 ReductionFunc->setAttributes(FuncAttrs);
4651 ReductionFunc->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
4652 ReductionFunc->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
4653 BasicBlock *EntryBB =
4654 BasicBlock::Create(Context&: M.getContext(), Name: "entry", Parent: ReductionFunc);
4655 Builder.SetInsertPoint(EntryBB);
4656 Builder.SetCurrentDebugLocation(llvm::DebugLoc());
4657
4658 // Need to alloca memory here and deal with the pointers before getting
4659 // LHS/RHS pointers out
4660 Value *LHSArrayPtr = nullptr;
4661 Value *RHSArrayPtr = nullptr;
4662 Argument *Arg0 = ReductionFunc->getArg(i: 0);
4663 Argument *Arg1 = ReductionFunc->getArg(i: 1);
4664 Type *Arg0Type = Arg0->getType();
4665 Type *Arg1Type = Arg1->getType();
4666
4667 Value *LHSAlloca =
4668 Builder.CreateAlloca(Ty: Arg0Type, ArraySize: nullptr, Name: Arg0->getName() + ".addr");
4669 Value *RHSAlloca =
4670 Builder.CreateAlloca(Ty: Arg1Type, ArraySize: nullptr, Name: Arg1->getName() + ".addr");
4671 Value *LHSAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4672 V: LHSAlloca, DestTy: Arg0Type, Name: LHSAlloca->getName() + ".ascast");
4673 Value *RHSAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4674 V: RHSAlloca, DestTy: Arg1Type, Name: RHSAlloca->getName() + ".ascast");
4675 Builder.CreateStore(Val: Arg0, Ptr: LHSAddrCast);
4676 Builder.CreateStore(Val: Arg1, Ptr: RHSAddrCast);
4677 LHSArrayPtr = Builder.CreateLoad(Ty: Arg0Type, Ptr: LHSAddrCast);
4678 RHSArrayPtr = Builder.CreateLoad(Ty: Arg1Type, Ptr: RHSAddrCast);
4679
4680 Type *RedArrayTy = ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: ReductionInfos.size());
4681 Type *IndexTy = Builder.getIndexTy(
4682 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
4683 SmallVector<Value *> LHSPtrs, RHSPtrs;
4684 for (auto En : enumerate(First&: ReductionInfos)) {
4685 const ReductionInfo &RI = En.value();
4686 Value *RHSI8PtrPtr = Builder.CreateInBoundsGEP(
4687 Ty: RedArrayTy, Ptr: RHSArrayPtr,
4688 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
4689 Value *RHSI8Ptr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: RHSI8PtrPtr);
4690 Value *RHSPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
4691 V: RHSI8Ptr, DestTy: RI.PrivateVariable->getType(),
4692 Name: RHSI8Ptr->getName() + ".ascast");
4693
4694 Value *LHSI8PtrPtr = Builder.CreateInBoundsGEP(
4695 Ty: RedArrayTy, Ptr: LHSArrayPtr,
4696 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
4697 Value *LHSI8Ptr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: LHSI8PtrPtr);
4698 Value *LHSPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
4699 V: LHSI8Ptr, DestTy: RI.Variable->getType(), Name: LHSI8Ptr->getName() + ".ascast");
4700
4701 if (ReductionGenCBKind == ReductionGenCBKind::Clang) {
4702 LHSPtrs.emplace_back(Args&: LHSPtr);
4703 RHSPtrs.emplace_back(Args&: RHSPtr);
4704 } else {
4705 Value *LHS = LHSPtr;
4706 Value *RHS = RHSPtr;
4707
4708 if (!IsByRef.empty() && !IsByRef[En.index()]) {
4709 LHS = Builder.CreateLoad(Ty: RI.ElementType, Ptr: LHSPtr);
4710 RHS = Builder.CreateLoad(Ty: RI.ElementType, Ptr: RHSPtr);
4711 }
4712
4713 Value *Reduced;
4714 InsertPointOrErrorTy AfterIP =
4715 RI.ReductionGen(Builder.saveIP(), LHS, RHS, Reduced);
4716 if (!AfterIP)
4717 return AfterIP.takeError();
4718 if (!Builder.GetInsertBlock())
4719 return ReductionFunc;
4720
4721 Builder.restoreIP(IP: *AfterIP);
4722
4723 if (!IsByRef.empty() && !IsByRef[En.index()])
4724 Builder.CreateStore(Val: Reduced, Ptr: LHSPtr);
4725 }
4726 }
4727
4728 if (ReductionGenCBKind == ReductionGenCBKind::Clang)
4729 for (auto En : enumerate(First&: ReductionInfos)) {
4730 unsigned Index = En.index();
4731 const ReductionInfo &RI = En.value();
4732 Value *LHSFixupPtr, *RHSFixupPtr;
4733 Builder.restoreIP(IP: RI.ReductionGenClang(
4734 Builder.saveIP(), Index, &LHSFixupPtr, &RHSFixupPtr, ReductionFunc));
4735
4736 // Fix the CallBack code genereated to use the correct Values for the LHS
4737 // and RHS
4738 LHSFixupPtr->replaceUsesWithIf(
4739 New: LHSPtrs[Index], ShouldReplace: [ReductionFunc](const Use &U) {
4740 return cast<Instruction>(Val: U.getUser())->getParent()->getParent() ==
4741 ReductionFunc;
4742 });
4743 RHSFixupPtr->replaceUsesWithIf(
4744 New: RHSPtrs[Index], ShouldReplace: [ReductionFunc](const Use &U) {
4745 return cast<Instruction>(Val: U.getUser())->getParent()->getParent() ==
4746 ReductionFunc;
4747 });
4748 }
4749
4750 Builder.CreateRetVoid();
4751 // Compiling with `-O0`, `alloca`s emitted in non-entry blocks are not hoisted
4752 // to the entry block (this is dones for higher opt levels by later passes in
4753 // the pipeline). This has caused issues because non-entry `alloca`s force the
4754 // function to use dynamic stack allocations and we might run out of scratch
4755 // memory.
4756 hoistNonEntryAllocasToEntryBlock(Func: ReductionFunc);
4757
4758 return ReductionFunc;
4759}
4760
4761static void
4762checkReductionInfos(ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos,
4763 bool IsGPU) {
4764 for (const OpenMPIRBuilder::ReductionInfo &RI : ReductionInfos) {
4765 (void)RI;
4766 assert(RI.Variable && "expected non-null variable");
4767 assert(RI.PrivateVariable && "expected non-null private variable");
4768 assert((RI.ReductionGen || RI.ReductionGenClang) &&
4769 "expected non-null reduction generator callback");
4770 if (!IsGPU) {
4771 assert(
4772 RI.Variable->getType() == RI.PrivateVariable->getType() &&
4773 "expected variables and their private equivalents to have the same "
4774 "type");
4775 }
4776 assert(RI.Variable->getType()->isPointerTy() &&
4777 "expected variables to be pointers");
4778 }
4779}
4780
4781// The atomic cross-team reduction fast path applies when every reduction in the
4782// set can be represented by an atomicrmw. Clang only populates it for scalar
4783// reductions with a supported atomic operator.
4784static bool isAtomicableReductionSet(
4785 ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos) {
4786 return all_of(Range&: ReductionInfos, P: [](const OpenMPIRBuilder::ReductionInfo &RI) {
4787 return static_cast<bool>(RI.AtomicReductionGen);
4788 });
4789}
4790
4791OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createReductionsGPU(
4792 const LocationDescription &Loc, InsertPointTy AllocaIP,
4793 InsertPointTy CodeGenIP, ArrayRef<ReductionInfo> ReductionInfos,
4794 ArrayRef<bool> IsByRef, bool IsNoWait, bool IsTeamsReduction, bool IsSPMD,
4795 ReductionGenCBKind ReductionGenCBKind, std::optional<omp::GV> GridValue,
4796 Value *SrcLocInfo) {
4797 if (!updateToLocation(Loc))
4798 return InsertPointTy();
4799 Builder.restoreIP(IP: CodeGenIP);
4800 checkReductionInfos(ReductionInfos, /*IsGPU*/ true);
4801 LLVMContext &Ctx = M.getContext();
4802
4803 // Source location for the ident struct
4804 if (!SrcLocInfo) {
4805 uint32_t SrcLocStrSize;
4806 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
4807 SrcLocInfo = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
4808 }
4809
4810 if (ReductionInfos.size() == 0)
4811 return Builder.saveIP();
4812
4813 BasicBlock *ContinuationBlock = nullptr;
4814 if (ReductionGenCBKind != ReductionGenCBKind::Clang) {
4815 // Copied code from createReductions
4816 BasicBlock *InsertBlock = Loc.IP.getBlock();
4817 ContinuationBlock =
4818 InsertBlock->splitBasicBlock(I: Loc.IP.getPoint(), BBName: "reduce.finalize");
4819 InsertBlock->getTerminator()->eraseFromParent();
4820 Builder.SetInsertPoint(TheBB: InsertBlock, IP: InsertBlock->end());
4821 }
4822
4823 Function *CurFunc = Builder.GetInsertBlock()->getParent();
4824 AttributeList FuncAttrs;
4825 AttrBuilder AttrBldr(Ctx);
4826 for (auto Attr : CurFunc->getAttributes().getFnAttrs())
4827 AttrBldr.addAttribute(A: Attr);
4828 AttrBldr.removeAttribute(Val: Attribute::OptimizeNone);
4829 FuncAttrs = FuncAttrs.addFnAttributes(C&: Ctx, B: AttrBldr);
4830
4831 CodeGenIP = Builder.saveIP();
4832 Expected<Function *> ReductionResult = createReductionFunction(
4833 ReducerName: Builder.GetInsertBlock()->getParent()->getName(), ReductionInfos, IsByRef,
4834 ReductionGenCBKind, FuncAttrs);
4835 if (!ReductionResult)
4836 return ReductionResult.takeError();
4837 Function *ReductionFunc = *ReductionResult;
4838 Builder.restoreIP(IP: CodeGenIP);
4839
4840 // Set the grid value in the config needed for lowering later on
4841 if (GridValue.has_value())
4842 Config.setGridValue(GridValue.value());
4843 else
4844 Config.setGridValue(getGridValue(T, Kernel: ReductionFunc));
4845
4846 // Build res = __kmpc_reduce{_nowait}(<gtid>, <n>, sizeof(RedList),
4847 // RedList, shuffle_reduce_func, interwarp_copy_func);
4848 // or
4849 // Build res = __kmpc_reduce_teams_nowait_simple(<loc>, <gtid>, <lck>);
4850 Value *Res;
4851
4852 // 1. Build a list of reduction variables.
4853 // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
4854 auto Size = ReductionInfos.size();
4855 Type *PtrTy = PointerType::get(C&: Ctx, AddressSpace: Config.getDefaultTargetAS());
4856 Type *FuncPtrTy =
4857 Builder.getPtrTy(AddrSpace: M.getDataLayout().getProgramAddressSpace());
4858 Type *RedArrayTy = ArrayType::get(ElementType: PtrTy, NumElements: Size);
4859 CodeGenIP = Builder.saveIP();
4860 Builder.restoreIP(IP: AllocaIP);
4861 Value *ReductionListAlloca =
4862 Builder.CreateAlloca(Ty: RedArrayTy, ArraySize: nullptr, Name: ".omp.reduction.red_list");
4863 Value *ReductionList = Builder.CreatePointerBitCastOrAddrSpaceCast(
4864 V: ReductionListAlloca, DestTy: PtrTy, Name: ReductionListAlloca->getName() + ".ascast");
4865 Builder.restoreIP(IP: CodeGenIP);
4866 Type *IndexTy = Builder.getIndexTy(
4867 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
4868 for (auto En : enumerate(First&: ReductionInfos)) {
4869 const ReductionInfo &RI = En.value();
4870 Value *ElemPtr = Builder.CreateInBoundsGEP(
4871 Ty: RedArrayTy, Ptr: ReductionList,
4872 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
4873
4874 Value *PrivateVar = RI.PrivateVariable;
4875 bool IsByRefElem = !IsByRef.empty() && IsByRef[En.index()];
4876 if (IsByRefElem)
4877 PrivateVar = Builder.CreateLoad(Ty: RI.ElementType, Ptr: PrivateVar);
4878
4879 Value *CastElem =
4880 Builder.CreatePointerBitCastOrAddrSpaceCast(V: PrivateVar, DestTy: PtrTy);
4881 Builder.CreateStore(Val: CastElem, Ptr: ElemPtr);
4882 }
4883 CodeGenIP = Builder.saveIP();
4884 Expected<Function *> SarFunc = emitShuffleAndReduceFunction(
4885 ReductionInfos, ReduceFn: ReductionFunc, FuncAttrs, IsByRef);
4886
4887 if (!SarFunc)
4888 return SarFunc.takeError();
4889
4890 Expected<Function *> CopyResult =
4891 emitInterWarpCopyFunction(Loc, ReductionInfos, FuncAttrs, IsByRef);
4892 if (!CopyResult)
4893 return CopyResult.takeError();
4894 Function *WcFunc = *CopyResult;
4895 Builder.restoreIP(IP: CodeGenIP);
4896
4897 Value *RL = Builder.CreatePointerBitCastOrAddrSpaceCast(V: ReductionList, DestTy: PtrTy);
4898
4899 // NOTE: ReductionDataSize is passed as the reduce_data_size argument to
4900 // __kmpc_nvptx_parallel_reduce_nowait_v2, but the runtime implementations do
4901 // not currently use it. It is computed here conservatively as max(element
4902 // sizes) * N rather than the exact sum, which over-calculates the size for
4903 // mixed reduction types but is harmless given the argument is unused.
4904 // TODO: Consider dropping this computation if the runtime API is ever revised
4905 // to remove the unused parameter.
4906 unsigned MaxDataSize = 0;
4907 SmallVector<Type *> ReductionTypeArgs;
4908 for (auto En : enumerate(First&: ReductionInfos)) {
4909 // Use ByRefElementType for by-ref reductions so that MaxDataSize matches
4910 // the actual data size stored in the global reduction buffer, consistent
4911 // with the ReductionsBufferTy struct used for GEP offsets below.
4912 Type *RedTypeArg = (!IsByRef.empty() && IsByRef[En.index()])
4913 ? En.value().ByRefElementType
4914 : En.value().ElementType;
4915 auto Size = M.getDataLayout().getTypeStoreSize(Ty: RedTypeArg);
4916 if (Size > MaxDataSize)
4917 MaxDataSize = Size;
4918 ReductionTypeArgs.emplace_back(Args&: RedTypeArg);
4919 }
4920 Value *ReductionDataSize =
4921 Builder.getInt64(C: MaxDataSize * ReductionInfos.size());
4922
4923 // Helper function to copy thread-local data back to the original reduction
4924 // list.
4925 Function *CopyScratchToListFunc = nullptr;
4926 // Thread-local storage for the reduction variables.
4927 Value *ScratchForCopyBack = nullptr;
4928 // RL pointer to which the final value from the per-thread scratch should be
4929 // copied back. (Basically RL, appropriately casted if necessary.)
4930 Value *RLForCopyBack = RL;
4931
4932 bool IsAtomicReduction =
4933 IsTeamsReduction && isAtomicableReductionSet(ReductionInfos);
4934
4935 if (!IsTeamsReduction) {
4936 Value *SarFuncCast =
4937 Builder.CreatePointerBitCastOrAddrSpaceCast(V: *SarFunc, DestTy: FuncPtrTy);
4938 Value *WcFuncCast =
4939 Builder.CreatePointerBitCastOrAddrSpaceCast(V: WcFunc, DestTy: FuncPtrTy);
4940 Value *Args[] = {SrcLocInfo, ReductionDataSize, RL, SarFuncCast,
4941 WcFuncCast};
4942 Function *Pv2Ptr = getOrCreateRuntimeFunctionPtr(
4943 FnID: RuntimeFunction::OMPRTL___kmpc_nvptx_parallel_reduce_nowait_v2);
4944 Res = createRuntimeFunctionCall(Callee: Pv2Ptr, Args);
4945 } else if (IsAtomicReduction) {
4946 // Atomic cross-team reduction fast path: determine the team's main thread
4947 // that is later to fold its value atomically into the mapped variable.
4948 Function *IsMainThreadFn = getOrCreateRuntimeFunctionPtr(
4949 FnID: RuntimeFunction::OMPRTL___kmpc_is_team_main_thread);
4950 Res = createRuntimeFunctionCall(Callee: IsMainThreadFn, Args: {});
4951 } else {
4952 CodeGenIP = Builder.saveIP();
4953 StructType *ReductionsBufferTy = StructType::create(
4954 Context&: Ctx, Elements: ReductionTypeArgs, Name: "struct._globalized_locals_ty");
4955
4956 Expected<Function *> LtGCFunc = emitListToGlobalCopyFunction(
4957 ReductionInfos, ReductionsBufferTy, FuncAttrs, IsByRef);
4958 if (!LtGCFunc)
4959 return LtGCFunc.takeError();
4960
4961 Expected<Function *> GtLCFunc = emitGlobalToListCopyFunction(
4962 ReductionInfos, ReductionsBufferTy, FuncAttrs, IsByRef);
4963 if (!GtLCFunc)
4964 return GtLCFunc.takeError();
4965
4966 Expected<Function *> GtLRFunc = emitGlobalToListReduceFunction(
4967 ReductionInfos, ReduceFn: ReductionFunc, ReductionsBufferTy, FuncAttrs, IsByRef);
4968 if (!GtLRFunc)
4969 return GtLRFunc.takeError();
4970
4971 Builder.restoreIP(IP: CodeGenIP);
4972
4973 // The runtime's cross-team final aggregate uses the storage pointed at by
4974 // its reduce-list argument as per-thread scratch. When the surrounding
4975 // kernel is already in SPMD execution mode, clang emitted each reduction
4976 // private as a per-thread `alloca addrspace(5)`, so the original red_list
4977 // (RL) is already per-thread and nothing else is needed.
4978 //
4979 // When the kernel is in Non-SPMD execution mode at codegen time, clang's
4980 // Generic-mode globalization put the reduction private into team-shared
4981 // LDS. OpenMPOpt may later upgrade the kernel to Generic-SPMD, at which
4982 // point all threads of the last team would race on the shared LDS slot.
4983 // Emit a per-thread scratch buffer and a per-thread RL, copy the team-local
4984 // value in, and hand the per-thread RL to the runtime instead. The writer
4985 // thread copies the final value from that per-thread scratch back to RL
4986 // before running the existing combine path below.
4987
4988 // Thread-local RL (might need localization below before being passed to the
4989 // runtime).
4990 Value *RuntimeRL = RL;
4991
4992 if (!IsSPMD) {
4993 CodeGenIP = Builder.saveIP();
4994 Builder.restoreIP(IP: AllocaIP);
4995 // Allocate thread-local buffer for the reduction variables.
4996 Value *PerThreadScratchAlloca = Builder.CreateAlloca(
4997 Ty: ReductionsBufferTy, /*ArraySize=*/nullptr, Name: ".omp.reduction.scratch");
4998 Value *PerThreadScratch = Builder.CreatePointerBitCastOrAddrSpaceCast(
4999 V: PerThreadScratchAlloca, DestTy: PtrTy,
5000 Name: PerThreadScratchAlloca->getName() + ".ascast");
5001 // Allocate thread-local buffer for the pointers to the reduction
5002 // variables.
5003 Value *PerThreadRedListAlloca =
5004 Builder.CreateAlloca(Ty: RedArrayTy, /*ArraySize=*/nullptr,
5005 Name: ".omp.reduction.per_thread_red_list");
5006 RuntimeRL = Builder.CreatePointerBitCastOrAddrSpaceCast(
5007 V: PerThreadRedListAlloca, DestTy: PtrTy,
5008 Name: PerThreadRedListAlloca->getName() + ".ascast");
5009 Builder.restoreIP(IP: CodeGenIP);
5010
5011 // Iterate over the reduction variables and copy the team-local value to
5012 // the thread-local buffer.
5013 for (auto En : enumerate(First&: ReductionInfos)) {
5014 const ReductionInfo &RI = En.value();
5015 bool IsByRefElem = !IsByRef.empty() && IsByRef[En.index()];
5016
5017 Value *FieldPtr = Builder.CreateConstInBoundsGEP2_32(
5018 Ty: ReductionsBufferTy, Ptr: PerThreadScratch, Idx0: 0, Idx1: En.index());
5019 Value *Slot = Builder.CreateConstInBoundsGEP2_32(Ty: RedArrayTy, Ptr: RuntimeRL,
5020 Idx0: 0, Idx1: En.index());
5021
5022 Value *RuntimeListEntry = FieldPtr;
5023 if (IsByRefElem && RI.DataPtrPtrGen) {
5024 Value *SrcDescriptor =
5025 Builder.CreateLoad(Ty: RI.ElementType, Ptr: RI.PrivateVariable);
5026 Expected<Value *> Descriptor = createReductionDescriptorCopy(
5027 AllocaIP, RI, DataPtr: FieldPtr, SrcDescriptorAddr: SrcDescriptor, DescriptorPtrTy: PtrTy);
5028 if (!Descriptor)
5029 return Descriptor.takeError();
5030 RuntimeListEntry = *Descriptor;
5031 }
5032 Builder.CreateStore(Val: RuntimeListEntry, Ptr: Slot);
5033 }
5034 // The copy helpers were emitted with default-AS (AS 0) pointer params
5035 // (see emitListToGlobalCopyFunction / emitGlobalToListCopyFunction),
5036 // but PerThreadScratch and RL live in the target's default AS, which
5037 // is non-zero on e.g. SPIRV. (See Config.getDefaultTargetAS().)
5038 Type *CopyArg0Ty = (*LtGCFunc)->getFunctionType()->getParamType(i: 0);
5039 Type *CopyArg2Ty = (*LtGCFunc)->getFunctionType()->getParamType(i: 2);
5040 ScratchForCopyBack = Builder.CreatePointerBitCastOrAddrSpaceCast(
5041 V: PerThreadScratch, DestTy: CopyArg0Ty);
5042 RLForCopyBack =
5043 Builder.CreatePointerBitCastOrAddrSpaceCast(V: RL, DestTy: CopyArg2Ty);
5044 // Use index 0 because there is no array of target values to index into,
5045 // there is only one thread-local memory slot.
5046 // restoreIP above left a stale/empty debug location; this inlinable call
5047 // to a debug-info-bearing helper needs one or the verifier rejects the
5048 // module ("!dbg attachment points at wrong subprogram") after inlining.
5049 Builder.SetCurrentDebugLocation(Loc.DL);
5050 Builder.CreateCall(
5051 Callee: *LtGCFunc, Args: {ScratchForCopyBack, Builder.getInt32(C: 0), RLForCopyBack});
5052 CopyScratchToListFunc = *GtLCFunc;
5053 }
5054
5055 Value *Args3[] = {SrcLocInfo, RuntimeRL, *SarFunc, WcFunc,
5056 *LtGCFunc, *GtLCFunc, *GtLRFunc};
5057
5058 Function *TeamsReduceFn = getOrCreateRuntimeFunctionPtr(
5059 FnID: RuntimeFunction::OMPRTL___kmpc_gpu_xteam_reduce_nowait);
5060 Res = createRuntimeFunctionCall(Callee: TeamsReduceFn, Args: Args3);
5061 }
5062
5063 // 5. Build if (res == 1)
5064 BasicBlock *ExitBB = BasicBlock::Create(Context&: Ctx, Name: ".omp.reduction.done");
5065 BasicBlock *ThenBB = BasicBlock::Create(Context&: Ctx, Name: ".omp.reduction.then");
5066 Value *Cond = Builder.CreateICmpEQ(LHS: Res, RHS: Builder.getInt32(C: 1));
5067 Builder.CreateCondBr(Cond, True: ThenBB, False: ExitBB);
5068
5069 // 6. Build then branch: where we have reduced values in the master
5070 // thread in each team.
5071 // __kmpc_end_reduce{_nowait}(<gtid>);
5072 // break;
5073 emitBlock(BB: ThenBB, CurFn: CurFunc);
5074
5075 // Copy the writer thread's per-thread scratch result back into the original
5076 // red-list storage before the existing combine path reads RI.PrivateVariable.
5077 // Set a debug location: this inlinable call to a debug-info-bearing helper
5078 // needs one or the verifier rejects the module after inlining.
5079 if (ScratchForCopyBack) {
5080 Builder.SetCurrentDebugLocation(Loc.DL);
5081 Builder.CreateCall(
5082 Callee: CopyScratchToListFunc,
5083 Args: {ScratchForCopyBack, Builder.getInt32(C: 0), RLForCopyBack});
5084 }
5085
5086 // Add emission of __kmpc_end_reduce{_nowait}(<gtid>);
5087 for (auto En : enumerate(First&: ReductionInfos)) {
5088 const ReductionInfo &RI = En.value();
5089
5090 // Atomic cross-team fast path: each team's main thread folds its
5091 // team-reduced value directly into the mapped reduction variable with a
5092 // single atomicrmw.
5093 if (IsAtomicReduction) {
5094 InsertPointOrErrorTy AfterIP = RI.AtomicReductionGen(
5095 Builder.saveIP(), RI.ElementType, RI.Variable, RI.PrivateVariable);
5096 if (!AfterIP)
5097 return AfterIP.takeError();
5098 Builder.restoreIP(IP: *AfterIP);
5099 continue;
5100 }
5101
5102 Type *ValueType = RI.ElementType;
5103 Value *RedValue = RI.Variable;
5104
5105 Value *RHS =
5106 Builder.CreatePointerBitCastOrAddrSpaceCast(V: RI.PrivateVariable, DestTy: PtrTy);
5107
5108 if (ReductionGenCBKind == ReductionGenCBKind::Clang) {
5109 Value *LHSPtr, *RHSPtr;
5110 Builder.restoreIP(IP: RI.ReductionGenClang(Builder.saveIP(), En.index(),
5111 &LHSPtr, &RHSPtr, CurFunc));
5112
5113 // Fix the CallBack code genereated to use the correct Values for the LHS
5114 // and RHS. Cast to match types before replacing (necessary to handle
5115 // different address spaces).
5116 if (LHSPtr->getType() != RedValue->getType())
5117 RedValue = Builder.CreatePointerBitCastOrAddrSpaceCast(
5118 V: RedValue, DestTy: LHSPtr->getType());
5119 if (RHSPtr->getType() != RHS->getType())
5120 RHS =
5121 Builder.CreatePointerBitCastOrAddrSpaceCast(V: RHS, DestTy: RHSPtr->getType());
5122
5123 LHSPtr->replaceUsesWithIf(New: RedValue, ShouldReplace: [ReductionFunc](const Use &U) {
5124 return cast<Instruction>(Val: U.getUser())->getParent()->getParent() ==
5125 ReductionFunc;
5126 });
5127 RHSPtr->replaceUsesWithIf(New: RHS, ShouldReplace: [ReductionFunc](const Use &U) {
5128 return cast<Instruction>(Val: U.getUser())->getParent()->getParent() ==
5129 ReductionFunc;
5130 });
5131 } else {
5132 if (IsByRef.empty() || !IsByRef[En.index()]) {
5133 RedValue = Builder.CreateLoad(Ty: ValueType, Ptr: RI.Variable,
5134 Name: "red.value." + Twine(En.index()));
5135 }
5136 Value *PrivateRedValue = Builder.CreateLoad(
5137 Ty: ValueType, Ptr: RHS, Name: "red.private.value" + Twine(En.index()));
5138 Value *Reduced;
5139 InsertPointOrErrorTy AfterIP =
5140 RI.ReductionGen(Builder.saveIP(), RedValue, PrivateRedValue, Reduced);
5141 if (!AfterIP)
5142 return AfterIP.takeError();
5143 Builder.restoreIP(IP: *AfterIP);
5144
5145 if (!IsByRef.empty() && !IsByRef[En.index()])
5146 Builder.CreateStore(Val: Reduced, Ptr: RI.Variable);
5147 }
5148 }
5149 emitBlock(BB: ExitBB, CurFn: CurFunc);
5150 if (ContinuationBlock) {
5151 Builder.CreateBr(Dest: ContinuationBlock);
5152 Builder.SetInsertPoint(ContinuationBlock);
5153 }
5154 Config.setEmitLLVMUsed();
5155
5156 return Builder.saveIP();
5157}
5158
5159static Function *getFreshReductionFunc(Module &M) {
5160 Type *VoidTy = Type::getVoidTy(C&: M.getContext());
5161 Type *Int8PtrTy = PointerType::getUnqual(C&: M.getContext());
5162 auto *FuncTy =
5163 FunctionType::get(Result: VoidTy, Params: {Int8PtrTy, Int8PtrTy}, /* IsVarArg */ isVarArg: false);
5164 return Function::Create(Ty: FuncTy, Linkage: GlobalVariable::InternalLinkage,
5165 N: ".omp.reduction.func", M: &M);
5166}
5167
5168static Error populateReductionFunction(
5169 Function *ReductionFunc,
5170 ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos,
5171 IRBuilder<> &Builder, ArrayRef<bool> IsByRef, bool IsGPU) {
5172 IRBuilder<>::InsertPointGuard IPG(Builder);
5173 Module *Module = ReductionFunc->getParent();
5174 BasicBlock *ReductionFuncBlock =
5175 BasicBlock::Create(Context&: Module->getContext(), Name: "", Parent: ReductionFunc);
5176 Builder.SetInsertPoint(ReductionFuncBlock);
5177 Builder.SetCurrentDebugLocation(llvm::DebugLoc());
5178 Value *LHSArrayPtr = nullptr;
5179 Value *RHSArrayPtr = nullptr;
5180 if (IsGPU) {
5181 // Need to alloca memory here and deal with the pointers before getting
5182 // LHS/RHS pointers out
5183 //
5184 Argument *Arg0 = ReductionFunc->getArg(i: 0);
5185 Argument *Arg1 = ReductionFunc->getArg(i: 1);
5186 Type *Arg0Type = Arg0->getType();
5187 Type *Arg1Type = Arg1->getType();
5188
5189 Value *LHSAlloca =
5190 Builder.CreateAlloca(Ty: Arg0Type, ArraySize: nullptr, Name: Arg0->getName() + ".addr");
5191 Value *RHSAlloca =
5192 Builder.CreateAlloca(Ty: Arg1Type, ArraySize: nullptr, Name: Arg1->getName() + ".addr");
5193 Value *LHSAddrCast =
5194 Builder.CreatePointerBitCastOrAddrSpaceCast(V: LHSAlloca, DestTy: Arg0Type);
5195 Value *RHSAddrCast =
5196 Builder.CreatePointerBitCastOrAddrSpaceCast(V: RHSAlloca, DestTy: Arg1Type);
5197 Builder.CreateStore(Val: Arg0, Ptr: LHSAddrCast);
5198 Builder.CreateStore(Val: Arg1, Ptr: RHSAddrCast);
5199 LHSArrayPtr = Builder.CreateLoad(Ty: Arg0Type, Ptr: LHSAddrCast);
5200 RHSArrayPtr = Builder.CreateLoad(Ty: Arg1Type, Ptr: RHSAddrCast);
5201 } else {
5202 LHSArrayPtr = ReductionFunc->getArg(i: 0);
5203 RHSArrayPtr = ReductionFunc->getArg(i: 1);
5204 }
5205
5206 unsigned NumReductions = ReductionInfos.size();
5207 Type *RedArrayTy = ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: NumReductions);
5208
5209 for (auto En : enumerate(First&: ReductionInfos)) {
5210 const OpenMPIRBuilder::ReductionInfo &RI = En.value();
5211 Value *LHSI8PtrPtr = Builder.CreateConstInBoundsGEP2_64(
5212 Ty: RedArrayTy, Ptr: LHSArrayPtr, Idx0: 0, Idx1: En.index());
5213 Value *LHSI8Ptr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: LHSI8PtrPtr);
5214 Value *LHSPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
5215 V: LHSI8Ptr, DestTy: RI.Variable->getType());
5216 Value *LHS = Builder.CreateLoad(Ty: RI.ElementType, Ptr: LHSPtr);
5217 Value *RHSI8PtrPtr = Builder.CreateConstInBoundsGEP2_64(
5218 Ty: RedArrayTy, Ptr: RHSArrayPtr, Idx0: 0, Idx1: En.index());
5219 Value *RHSI8Ptr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: RHSI8PtrPtr);
5220 Value *RHSPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
5221 V: RHSI8Ptr, DestTy: RI.PrivateVariable->getType());
5222 Value *RHS = Builder.CreateLoad(Ty: RI.ElementType, Ptr: RHSPtr);
5223 Value *Reduced;
5224 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
5225 RI.ReductionGen(Builder.saveIP(), LHS, RHS, Reduced);
5226 if (!AfterIP)
5227 return AfterIP.takeError();
5228
5229 Builder.restoreIP(IP: *AfterIP);
5230 // TODO: Consider flagging an error.
5231 if (!Builder.GetInsertBlock())
5232 return Error::success();
5233
5234 // store is inside of the reduction region when using by-ref
5235 if (!IsByRef[En.index()])
5236 Builder.CreateStore(Val: Reduced, Ptr: LHSPtr);
5237 }
5238 Builder.CreateRetVoid();
5239 return Error::success();
5240}
5241
5242OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createReductions(
5243 const LocationDescription &Loc, InsertPointTy AllocaIP,
5244 ArrayRef<ReductionInfo> ReductionInfos, ArrayRef<bool> IsByRef,
5245 bool IsNoWait, bool IsTeamsReduction) {
5246 assert(ReductionInfos.size() == IsByRef.size());
5247 if (Config.isGPU())
5248 return createReductionsGPU(Loc, AllocaIP, CodeGenIP: Builder.saveIP(), ReductionInfos,
5249 IsByRef, IsNoWait, IsTeamsReduction);
5250
5251 checkReductionInfos(ReductionInfos, /*IsGPU*/ false);
5252
5253 if (!updateToLocation(Loc))
5254 return InsertPointTy();
5255
5256 if (ReductionInfos.size() == 0)
5257 return Builder.saveIP();
5258
5259 BasicBlock *InsertBlock = Loc.IP.getBlock();
5260 BasicBlock *ContinuationBlock =
5261 InsertBlock->splitBasicBlock(I: Loc.IP.getPoint(), BBName: "reduce.finalize");
5262 InsertBlock->getTerminator()->eraseFromParent();
5263
5264 // Create and populate array of type-erased pointers to private reduction
5265 // values.
5266 unsigned NumReductions = ReductionInfos.size();
5267 Type *RedArrayTy = ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: NumReductions);
5268 Builder.SetInsertPoint(AllocaIP.getBlock()->getTerminator());
5269 Value *RedArray = Builder.CreateAlloca(Ty: RedArrayTy, ArraySize: nullptr, Name: "red.array");
5270
5271 Builder.SetInsertPoint(TheBB: InsertBlock, IP: InsertBlock->end());
5272
5273 for (auto En : enumerate(First&: ReductionInfos)) {
5274 unsigned Index = En.index();
5275 const ReductionInfo &RI = En.value();
5276 Value *RedArrayElemPtr = Builder.CreateConstInBoundsGEP2_64(
5277 Ty: RedArrayTy, Ptr: RedArray, Idx0: 0, Idx1: Index, Name: "red.array.elem." + Twine(Index));
5278 Builder.CreateStore(Val: RI.PrivateVariable, Ptr: RedArrayElemPtr);
5279 }
5280
5281 // Emit a call to the runtime function that orchestrates the reduction.
5282 // Declare the reduction function in the process.
5283 Type *IndexTy = Builder.getIndexTy(
5284 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
5285 Function *Func = Builder.GetInsertBlock()->getParent();
5286 Module *Module = Func->getParent();
5287 uint32_t SrcLocStrSize;
5288 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
5289 bool CanGenerateAtomic = all_of(Range&: ReductionInfos, P: [](const ReductionInfo &RI) {
5290 return RI.AtomicReductionGen;
5291 });
5292 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize,
5293 LocFlags: CanGenerateAtomic
5294 ? IdentFlag::OMP_IDENT_FLAG_ATOMIC_REDUCE
5295 : IdentFlag(0));
5296 Value *ThreadId = getOrCreateThreadID(Ident);
5297 Constant *NumVariables = Builder.getInt32(C: NumReductions);
5298 const DataLayout &DL = Module->getDataLayout();
5299 unsigned RedArrayByteSize = DL.getTypeStoreSize(Ty: RedArrayTy);
5300 Constant *RedArraySize = ConstantInt::get(Ty: IndexTy, V: RedArrayByteSize);
5301 Function *ReductionFunc = getFreshReductionFunc(M&: *Module);
5302 Value *Lock = getOMPCriticalRegionLock(CriticalName: ".reduction");
5303 Function *ReduceFunc = getOrCreateRuntimeFunctionPtr(
5304 FnID: IsNoWait ? RuntimeFunction::OMPRTL___kmpc_reduce_nowait
5305 : RuntimeFunction::OMPRTL___kmpc_reduce);
5306 CallInst *ReduceCall =
5307 createRuntimeFunctionCall(Callee: ReduceFunc,
5308 Args: {Ident, ThreadId, NumVariables, RedArraySize,
5309 RedArray, ReductionFunc, Lock},
5310 Name: "reduce");
5311
5312 // Create final reduction entry blocks for the atomic and non-atomic case.
5313 // Emit IR that dispatches control flow to one of the blocks based on the
5314 // reduction supporting the atomic mode.
5315 BasicBlock *NonAtomicRedBlock =
5316 BasicBlock::Create(Context&: Module->getContext(), Name: "reduce.switch.nonatomic", Parent: Func);
5317 BasicBlock *AtomicRedBlock =
5318 BasicBlock::Create(Context&: Module->getContext(), Name: "reduce.switch.atomic", Parent: Func);
5319 SwitchInst *Switch =
5320 Builder.CreateSwitch(V: ReduceCall, Dest: ContinuationBlock, /* NumCases */ 2);
5321 Switch->addCase(OnVal: Builder.getInt32(C: 1), Dest: NonAtomicRedBlock);
5322 Switch->addCase(OnVal: Builder.getInt32(C: 2), Dest: AtomicRedBlock);
5323
5324 // Populate the non-atomic reduction using the elementwise reduction function.
5325 // This loads the elements from the global and private variables and reduces
5326 // them before storing back the result to the global variable.
5327 Builder.SetInsertPoint(NonAtomicRedBlock);
5328 for (auto En : enumerate(First&: ReductionInfos)) {
5329 const ReductionInfo &RI = En.value();
5330 Type *ValueType = RI.ElementType;
5331 // We have one less load for by-ref case because that load is now inside of
5332 // the reduction region
5333 Value *RedValue = RI.Variable;
5334 if (!IsByRef[En.index()]) {
5335 RedValue = Builder.CreateLoad(Ty: ValueType, Ptr: RI.Variable,
5336 Name: "red.value." + Twine(En.index()));
5337 }
5338 Value *PrivateRedValue =
5339 Builder.CreateLoad(Ty: ValueType, Ptr: RI.PrivateVariable,
5340 Name: "red.private.value." + Twine(En.index()));
5341 Value *Reduced;
5342 InsertPointOrErrorTy AfterIP =
5343 RI.ReductionGen(Builder.saveIP(), RedValue, PrivateRedValue, Reduced);
5344 if (!AfterIP)
5345 return AfterIP.takeError();
5346 Builder.restoreIP(IP: *AfterIP);
5347
5348 if (!Builder.GetInsertBlock())
5349 return InsertPointTy();
5350 // for by-ref case, the load is inside of the reduction region
5351 if (!IsByRef[En.index()])
5352 Builder.CreateStore(Val: Reduced, Ptr: RI.Variable);
5353 }
5354 Function *EndReduceFunc = getOrCreateRuntimeFunctionPtr(
5355 FnID: IsNoWait ? RuntimeFunction::OMPRTL___kmpc_end_reduce_nowait
5356 : RuntimeFunction::OMPRTL___kmpc_end_reduce);
5357 createRuntimeFunctionCall(Callee: EndReduceFunc, Args: {Ident, ThreadId, Lock});
5358 Builder.CreateBr(Dest: ContinuationBlock);
5359
5360 // Populate the atomic reduction using the atomic elementwise reduction
5361 // function. There are no loads/stores here because they will be happening
5362 // inside the atomic elementwise reduction.
5363 Builder.SetInsertPoint(AtomicRedBlock);
5364 if (CanGenerateAtomic && llvm::none_of(Range&: IsByRef, P: [](bool P) { return P; })) {
5365 for (const ReductionInfo &RI : ReductionInfos) {
5366 InsertPointOrErrorTy AfterIP = RI.AtomicReductionGen(
5367 Builder.saveIP(), RI.ElementType, RI.Variable, RI.PrivateVariable);
5368 if (!AfterIP)
5369 return AfterIP.takeError();
5370 Builder.restoreIP(IP: *AfterIP);
5371 if (!Builder.GetInsertBlock())
5372 return InsertPointTy();
5373 }
5374 Builder.CreateBr(Dest: ContinuationBlock);
5375 } else {
5376 Builder.CreateUnreachable();
5377 }
5378
5379 // Populate the outlined reduction function using the elementwise reduction
5380 // function. Partial values are extracted from the type-erased array of
5381 // pointers to private variables.
5382 Error Err = populateReductionFunction(ReductionFunc, ReductionInfos, Builder,
5383 IsByRef, /*isGPU=*/IsGPU: false);
5384 if (Err)
5385 return Err;
5386
5387 if (!Builder.GetInsertBlock())
5388 return InsertPointTy();
5389
5390 Builder.SetInsertPoint(ContinuationBlock);
5391 return Builder.saveIP();
5392}
5393
5394OpenMPIRBuilder::InsertPointOrErrorTy
5395OpenMPIRBuilder::createMaster(const LocationDescription &Loc,
5396 BodyGenCallbackTy BodyGenCB,
5397 FinalizeCallbackTy FiniCB) {
5398 if (!updateToLocation(Loc))
5399 return Loc.IP;
5400
5401 Directive OMPD = Directive::OMPD_master;
5402 uint32_t SrcLocStrSize;
5403 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
5404 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
5405 Value *ThreadId = getOrCreateThreadID(Ident);
5406 Value *Args[] = {Ident, ThreadId};
5407
5408 Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_master);
5409 Instruction *EntryCall = createRuntimeFunctionCall(Callee: EntryRTLFn, Args);
5410
5411 Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_end_master);
5412 Instruction *ExitCall = createRuntimeFunctionCall(Callee: ExitRTLFn, Args);
5413
5414 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
5415 /*Conditional*/ true, /*hasFinalize*/ HasFinalize: true);
5416}
5417
5418OpenMPIRBuilder::InsertPointOrErrorTy
5419OpenMPIRBuilder::createMasked(const LocationDescription &Loc,
5420 BodyGenCallbackTy BodyGenCB,
5421 FinalizeCallbackTy FiniCB, Value *Filter) {
5422 if (!updateToLocation(Loc))
5423 return Loc.IP;
5424
5425 Directive OMPD = Directive::OMPD_masked;
5426 uint32_t SrcLocStrSize;
5427 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
5428 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
5429 Value *ThreadId = getOrCreateThreadID(Ident);
5430 Value *Args[] = {Ident, ThreadId, Filter};
5431 Value *ArgsEnd[] = {Ident, ThreadId};
5432
5433 Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_masked);
5434 Instruction *EntryCall = createRuntimeFunctionCall(Callee: EntryRTLFn, Args);
5435
5436 Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_end_masked);
5437 Instruction *ExitCall = createRuntimeFunctionCall(Callee: ExitRTLFn, Args: ArgsEnd);
5438
5439 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
5440 /*Conditional*/ true, /*hasFinalize*/ HasFinalize: true);
5441}
5442
5443static llvm::CallInst *emitNoUnwindRuntimeCall(IRBuilder<> &Builder,
5444 llvm::FunctionCallee Callee,
5445 ArrayRef<llvm::Value *> Args,
5446 const llvm::Twine &Name) {
5447 llvm::CallInst *Call = Builder.CreateCall(
5448 Callee, Args, OpBundles: SmallVector<llvm::OperandBundleDef, 1>(), Name);
5449 Call->setDoesNotThrow();
5450 return Call;
5451}
5452
5453// Expects input basic block is dominated by BeforeScanBB.
5454// Once Scan directive is encountered, the code after scan directive should be
5455// dominated by AfterScanBB. Scan directive splits the code sequence to
5456// scan and input phase. Based on whether inclusive or exclusive
5457// clause is used in the scan directive and whether input loop or scan loop
5458// is lowered, it adds jumps to input and scan phase. First Scan loop is the
5459// input loop and second is the scan loop. The code generated handles only
5460// inclusive scans now.
5461OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createScan(
5462 const LocationDescription &Loc, InsertPointTy AllocaIP,
5463 ArrayRef<llvm::Value *> ScanVars, ArrayRef<llvm::Type *> ScanVarsType,
5464 bool IsInclusive, ScanInfo *ScanRedInfo) {
5465 if (ScanRedInfo->OMPFirstScanLoop) {
5466 llvm::Error Err = emitScanBasedDirectiveDeclsIR(AllocaIP, ScanVars,
5467 ScanVarsType, ScanRedInfo);
5468 if (Err)
5469 return Err;
5470 }
5471 if (!updateToLocation(Loc))
5472 return Loc.IP;
5473
5474 llvm::Value *IV = ScanRedInfo->IV;
5475
5476 if (ScanRedInfo->OMPFirstScanLoop) {
5477 // Emit buffer[i] = red; at the end of the input phase.
5478 for (size_t i = 0; i < ScanVars.size(); i++) {
5479 Value *BuffPtr = (*(ScanRedInfo->ScanBuffPtrs))[ScanVars[i]];
5480 Value *Buff = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: BuffPtr);
5481 Type *DestTy = ScanVarsType[i];
5482 Value *Val = Builder.CreateInBoundsGEP(Ty: DestTy, Ptr: Buff, IdxList: IV, Name: "arrayOffset");
5483 Value *Src = Builder.CreateLoad(Ty: DestTy, Ptr: ScanVars[i]);
5484
5485 Builder.CreateStore(Val: Src, Ptr: Val);
5486 }
5487 }
5488 Builder.CreateBr(Dest: ScanRedInfo->OMPScanLoopExit);
5489 emitBlock(BB: ScanRedInfo->OMPScanDispatch,
5490 CurFn: Builder.GetInsertBlock()->getParent());
5491
5492 if (!ScanRedInfo->OMPFirstScanLoop) {
5493 IV = ScanRedInfo->IV;
5494 // Emit red = buffer[i]; at the entrance to the scan phase.
5495 // TODO: if exclusive scan, the red = buffer[i-1] needs to be updated.
5496 for (size_t i = 0; i < ScanVars.size(); i++) {
5497 Value *BuffPtr = (*(ScanRedInfo->ScanBuffPtrs))[ScanVars[i]];
5498 Value *Buff = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: BuffPtr);
5499 Type *DestTy = ScanVarsType[i];
5500 Value *SrcPtr =
5501 Builder.CreateInBoundsGEP(Ty: DestTy, Ptr: Buff, IdxList: IV, Name: "arrayOffset");
5502 Value *Src = Builder.CreateLoad(Ty: DestTy, Ptr: SrcPtr);
5503 Builder.CreateStore(Val: Src, Ptr: ScanVars[i]);
5504 }
5505 }
5506
5507 // TODO: Update it to CreateBr and remove dead blocks
5508 llvm::Value *CmpI = Builder.getInt1(V: true);
5509 if (ScanRedInfo->OMPFirstScanLoop == IsInclusive) {
5510 Builder.CreateCondBr(Cond: CmpI, True: ScanRedInfo->OMPBeforeScanBlock,
5511 False: ScanRedInfo->OMPAfterScanBlock);
5512 } else {
5513 Builder.CreateCondBr(Cond: CmpI, True: ScanRedInfo->OMPAfterScanBlock,
5514 False: ScanRedInfo->OMPBeforeScanBlock);
5515 }
5516 emitBlock(BB: ScanRedInfo->OMPAfterScanBlock,
5517 CurFn: Builder.GetInsertBlock()->getParent());
5518 Builder.SetInsertPoint(ScanRedInfo->OMPAfterScanBlock);
5519 return Builder.saveIP();
5520}
5521
5522Error OpenMPIRBuilder::emitScanBasedDirectiveDeclsIR(
5523 InsertPointTy AllocaIP, ArrayRef<Value *> ScanVars,
5524 ArrayRef<Type *> ScanVarsType, ScanInfo *ScanRedInfo) {
5525
5526 Builder.restoreIP(IP: AllocaIP);
5527 // Create the shared pointer at alloca IP.
5528 for (size_t i = 0; i < ScanVars.size(); i++) {
5529 llvm::Value *BuffPtr =
5530 Builder.CreateAlloca(Ty: Builder.getPtrTy(), ArraySize: nullptr, Name: "vla");
5531 (*(ScanRedInfo->ScanBuffPtrs))[ScanVars[i]] = BuffPtr;
5532 }
5533
5534 // Allocate temporary buffer by master thread
5535 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
5536 ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
5537 Builder.restoreIP(IP: CodeGenIP);
5538 Value *AllocSpan =
5539 Builder.CreateAdd(LHS: ScanRedInfo->Span, RHS: Builder.getInt32(C: 1));
5540 for (size_t i = 0; i < ScanVars.size(); i++) {
5541 Type *IntPtrTy = Builder.getInt32Ty();
5542 Constant *Allocsize = ConstantExpr::getSizeOf(Ty: ScanVarsType[i]);
5543 Allocsize = ConstantExpr::getTruncOrBitCast(C: Allocsize, Ty: IntPtrTy);
5544 Value *Buff = Builder.CreateMalloc(IntPtrTy, AllocTy: ScanVarsType[i], AllocSize: Allocsize,
5545 ArraySize: AllocSpan, MallocF: nullptr, Name: "arr");
5546 Builder.CreateStore(Val: Buff, Ptr: (*(ScanRedInfo->ScanBuffPtrs))[ScanVars[i]]);
5547 }
5548 return Error::success();
5549 };
5550 // TODO: Perform finalization actions for variables. This has to be
5551 // called for variables which have destructors/finalizers.
5552 auto FiniCB = [&](InsertPointTy CodeGenIP) { return llvm::Error::success(); };
5553
5554 Builder.SetInsertPoint(ScanRedInfo->OMPScanInit->getTerminator());
5555 llvm::Value *FilterVal = Builder.getInt32(C: 0);
5556 llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
5557 createMasked(Loc: Builder.saveIP(), BodyGenCB, FiniCB, Filter: FilterVal);
5558
5559 if (!AfterIP)
5560 return AfterIP.takeError();
5561 Builder.restoreIP(IP: *AfterIP);
5562 BasicBlock *InputBB = Builder.GetInsertBlock();
5563 if (InputBB->hasTerminator())
5564 Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator());
5565 AfterIP = createBarrier(Loc: Builder.saveIP(), Kind: llvm::omp::OMPD_barrier);
5566 if (!AfterIP)
5567 return AfterIP.takeError();
5568 Builder.restoreIP(IP: *AfterIP);
5569
5570 return Error::success();
5571}
5572
5573Error OpenMPIRBuilder::emitScanBasedDirectiveFinalsIR(
5574 ArrayRef<ReductionInfo> ReductionInfos, ScanInfo *ScanRedInfo) {
5575 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
5576 ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
5577 Builder.restoreIP(IP: CodeGenIP);
5578 for (ReductionInfo RedInfo : ReductionInfos) {
5579 Value *PrivateVar = RedInfo.PrivateVariable;
5580 Value *OrigVar = RedInfo.Variable;
5581 Value *BuffPtr = (*(ScanRedInfo->ScanBuffPtrs))[PrivateVar];
5582 Value *Buff = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: BuffPtr);
5583
5584 Type *SrcTy = RedInfo.ElementType;
5585 Value *Val = Builder.CreateInBoundsGEP(Ty: SrcTy, Ptr: Buff, IdxList: ScanRedInfo->Span,
5586 Name: "arrayOffset");
5587 Value *Src = Builder.CreateLoad(Ty: SrcTy, Ptr: Val);
5588
5589 Builder.CreateStore(Val: Src, Ptr: OrigVar);
5590 Builder.CreateFree(Source: Buff);
5591 }
5592 return Error::success();
5593 };
5594 // TODO: Perform finalization actions for variables. This has to be
5595 // called for variables which have destructors/finalizers.
5596 auto FiniCB = [&](InsertPointTy CodeGenIP) { return llvm::Error::success(); };
5597
5598 if (Instruction *TI = ScanRedInfo->OMPScanFinish->getTerminatorOrNull())
5599 Builder.SetInsertPoint(TI);
5600 else
5601 Builder.SetInsertPoint(ScanRedInfo->OMPScanFinish);
5602
5603 llvm::Value *FilterVal = Builder.getInt32(C: 0);
5604 llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
5605 createMasked(Loc: Builder.saveIP(), BodyGenCB, FiniCB, Filter: FilterVal);
5606
5607 if (!AfterIP)
5608 return AfterIP.takeError();
5609 Builder.restoreIP(IP: *AfterIP);
5610 BasicBlock *InputBB = Builder.GetInsertBlock();
5611 if (InputBB->hasTerminator())
5612 Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator());
5613 AfterIP = createBarrier(Loc: Builder.saveIP(), Kind: llvm::omp::OMPD_barrier);
5614 if (!AfterIP)
5615 return AfterIP.takeError();
5616 Builder.restoreIP(IP: *AfterIP);
5617 return Error::success();
5618}
5619
5620OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
5621 const LocationDescription &Loc,
5622 ArrayRef<llvm::OpenMPIRBuilder::ReductionInfo> ReductionInfos,
5623 ScanInfo *ScanRedInfo) {
5624
5625 if (!updateToLocation(Loc))
5626 return Loc.IP;
5627 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
5628 ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
5629 Builder.restoreIP(IP: CodeGenIP);
5630 Function *CurFn = Builder.GetInsertBlock()->getParent();
5631 // for (int k = 0; k <= ceil(log2(n)); ++k)
5632 llvm::BasicBlock *LoopBB =
5633 BasicBlock::Create(Context&: CurFn->getContext(), Name: "omp.outer.log.scan.body");
5634 llvm::BasicBlock *ExitBB =
5635 splitBB(Builder, CreateBranch: false, Name: "omp.outer.log.scan.exit");
5636 llvm::Function *F = llvm::Intrinsic::getOrInsertDeclaration(
5637 M: Builder.GetInsertBlock()->getModule(),
5638 id: (llvm::Intrinsic::ID)llvm::Intrinsic::log2, OverloadTys: Builder.getDoubleTy());
5639 llvm::BasicBlock *InputBB = Builder.GetInsertBlock();
5640 llvm::Value *Arg =
5641 Builder.CreateUIToFP(V: ScanRedInfo->Span, DestTy: Builder.getDoubleTy());
5642 llvm::Value *LogVal = emitNoUnwindRuntimeCall(Builder, Callee: F, Args: Arg, Name: "");
5643 F = llvm::Intrinsic::getOrInsertDeclaration(
5644 M: Builder.GetInsertBlock()->getModule(),
5645 id: (llvm::Intrinsic::ID)llvm::Intrinsic::ceil, OverloadTys: Builder.getDoubleTy());
5646 LogVal = emitNoUnwindRuntimeCall(Builder, Callee: F, Args: LogVal, Name: "");
5647 LogVal = Builder.CreateFPToUI(V: LogVal, DestTy: Builder.getInt32Ty());
5648 llvm::Value *NMin1 = Builder.CreateNUWSub(
5649 LHS: ScanRedInfo->Span,
5650 RHS: llvm::ConstantInt::get(Ty: ScanRedInfo->Span->getType(), V: 1));
5651 Builder.SetInsertPoint(InputBB);
5652 Builder.CreateBr(Dest: LoopBB);
5653 emitBlock(BB: LoopBB, CurFn);
5654 Builder.SetInsertPoint(LoopBB);
5655
5656 PHINode *Counter = Builder.CreatePHI(Ty: Builder.getInt32Ty(), NumReservedValues: 2);
5657 // size pow2k = 1;
5658 PHINode *Pow2K = Builder.CreatePHI(Ty: Builder.getInt32Ty(), NumReservedValues: 2);
5659 Counter->addIncoming(V: llvm::ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0),
5660 BB: InputBB);
5661 Pow2K->addIncoming(V: llvm::ConstantInt::get(Ty: Builder.getInt32Ty(), V: 1),
5662 BB: InputBB);
5663 // for (size i = n - 1; i >= 2 ^ k; --i)
5664 // tmp[i] op= tmp[i-pow2k];
5665 llvm::BasicBlock *InnerLoopBB =
5666 BasicBlock::Create(Context&: CurFn->getContext(), Name: "omp.inner.log.scan.body");
5667 llvm::BasicBlock *InnerExitBB =
5668 BasicBlock::Create(Context&: CurFn->getContext(), Name: "omp.inner.log.scan.exit");
5669 llvm::Value *CmpI = Builder.CreateICmpUGE(LHS: NMin1, RHS: Pow2K);
5670 Builder.CreateCondBr(Cond: CmpI, True: InnerLoopBB, False: InnerExitBB);
5671 emitBlock(BB: InnerLoopBB, CurFn);
5672 Builder.SetInsertPoint(InnerLoopBB);
5673 PHINode *IVal = Builder.CreatePHI(Ty: Builder.getInt32Ty(), NumReservedValues: 2);
5674 IVal->addIncoming(V: NMin1, BB: LoopBB);
5675 for (ReductionInfo RedInfo : ReductionInfos) {
5676 Value *ReductionVal = RedInfo.PrivateVariable;
5677 Value *BuffPtr = (*(ScanRedInfo->ScanBuffPtrs))[ReductionVal];
5678 Value *Buff = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: BuffPtr);
5679 Type *DestTy = RedInfo.ElementType;
5680 Value *IV = Builder.CreateAdd(LHS: IVal, RHS: Builder.getInt32(C: 1));
5681 Value *LHSPtr =
5682 Builder.CreateInBoundsGEP(Ty: DestTy, Ptr: Buff, IdxList: IV, Name: "arrayOffset");
5683 Value *OffsetIval = Builder.CreateNUWSub(LHS: IV, RHS: Pow2K);
5684 Value *RHSPtr =
5685 Builder.CreateInBoundsGEP(Ty: DestTy, Ptr: Buff, IdxList: OffsetIval, Name: "arrayOffset");
5686 Value *LHS = Builder.CreateLoad(Ty: DestTy, Ptr: LHSPtr);
5687 Value *RHS = Builder.CreateLoad(Ty: DestTy, Ptr: RHSPtr);
5688 llvm::Value *Result;
5689 InsertPointOrErrorTy AfterIP =
5690 RedInfo.ReductionGen(Builder.saveIP(), LHS, RHS, Result);
5691 if (!AfterIP)
5692 return AfterIP.takeError();
5693 Builder.CreateStore(Val: Result, Ptr: LHSPtr);
5694 }
5695 llvm::Value *NextIVal = Builder.CreateNUWSub(
5696 LHS: IVal, RHS: llvm::ConstantInt::get(Ty: Builder.getInt32Ty(), V: 1));
5697 IVal->addIncoming(V: NextIVal, BB: Builder.GetInsertBlock());
5698 CmpI = Builder.CreateICmpUGE(LHS: NextIVal, RHS: Pow2K);
5699 Builder.CreateCondBr(Cond: CmpI, True: InnerLoopBB, False: InnerExitBB);
5700 emitBlock(BB: InnerExitBB, CurFn);
5701 llvm::Value *Next = Builder.CreateNUWAdd(
5702 LHS: Counter, RHS: llvm::ConstantInt::get(Ty: Counter->getType(), V: 1));
5703 Counter->addIncoming(V: Next, BB: Builder.GetInsertBlock());
5704 // pow2k <<= 1;
5705 llvm::Value *NextPow2K = Builder.CreateShl(LHS: Pow2K, RHS: 1, Name: "", /*HasNUW=*/true);
5706 Pow2K->addIncoming(V: NextPow2K, BB: Builder.GetInsertBlock());
5707 llvm::Value *Cmp = Builder.CreateICmpNE(LHS: Next, RHS: LogVal);
5708 Builder.CreateCondBr(Cond: Cmp, True: LoopBB, False: ExitBB);
5709 Builder.SetInsertPoint(ExitBB->getFirstInsertionPt());
5710 return Error::success();
5711 };
5712
5713 // TODO: Perform finalization actions for variables. This has to be
5714 // called for variables which have destructors/finalizers.
5715 auto FiniCB = [&](InsertPointTy CodeGenIP) { return llvm::Error::success(); };
5716
5717 llvm::Value *FilterVal = Builder.getInt32(C: 0);
5718 llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
5719 createMasked(Loc: Builder.saveIP(), BodyGenCB, FiniCB, Filter: FilterVal);
5720
5721 if (!AfterIP)
5722 return AfterIP.takeError();
5723 Builder.restoreIP(IP: *AfterIP);
5724 AfterIP = createBarrier(Loc: Builder.saveIP(), Kind: llvm::omp::OMPD_barrier);
5725
5726 if (!AfterIP)
5727 return AfterIP.takeError();
5728 Builder.restoreIP(IP: *AfterIP);
5729 Error Err = emitScanBasedDirectiveFinalsIR(ReductionInfos, ScanRedInfo);
5730 if (Err)
5731 return Err;
5732
5733 return AfterIP;
5734}
5735
5736Error OpenMPIRBuilder::emitScanBasedDirectiveIR(
5737 llvm::function_ref<Error()> InputLoopGen,
5738 llvm::function_ref<Error(LocationDescription Loc)> ScanLoopGen,
5739 ScanInfo *ScanRedInfo) {
5740
5741 {
5742 // Emit loop with input phase:
5743 // for (i: 0..<num_iters>) {
5744 // <input phase>;
5745 // buffer[i] = red;
5746 // }
5747 ScanRedInfo->OMPFirstScanLoop = true;
5748 Error Err = InputLoopGen();
5749 if (Err)
5750 return Err;
5751 }
5752 {
5753 // Emit loop with scan phase:
5754 // for (i: 0..<num_iters>) {
5755 // red = buffer[i];
5756 // <scan phase>;
5757 // }
5758 ScanRedInfo->OMPFirstScanLoop = false;
5759 Error Err = ScanLoopGen(Builder.saveIP());
5760 if (Err)
5761 return Err;
5762 }
5763 return Error::success();
5764}
5765
5766void OpenMPIRBuilder::createScanBBs(ScanInfo *ScanRedInfo) {
5767 Function *Fun = Builder.GetInsertBlock()->getParent();
5768 ScanRedInfo->OMPScanDispatch =
5769 BasicBlock::Create(Context&: Fun->getContext(), Name: "omp.inscan.dispatch");
5770 ScanRedInfo->OMPAfterScanBlock =
5771 BasicBlock::Create(Context&: Fun->getContext(), Name: "omp.after.scan.bb");
5772 ScanRedInfo->OMPBeforeScanBlock =
5773 BasicBlock::Create(Context&: Fun->getContext(), Name: "omp.before.scan.bb");
5774 ScanRedInfo->OMPScanLoopExit =
5775 BasicBlock::Create(Context&: Fun->getContext(), Name: "omp.scan.loop.exit");
5776}
5777CanonicalLoopInfo *OpenMPIRBuilder::createLoopSkeleton(
5778 DebugLoc DL, Value *TripCount, Function *F, BasicBlock *PreInsertBefore,
5779 BasicBlock *PostInsertBefore, const Twine &Name, bool IsCollapsed) {
5780 Module *M = F->getParent();
5781 LLVMContext &Ctx = M->getContext();
5782 Type *IndVarTy = TripCount->getType();
5783
5784 // Create the basic block structure.
5785 BasicBlock *Preheader =
5786 BasicBlock::Create(Context&: Ctx, Name: "omp_" + Name + ".preheader", Parent: F, InsertBefore: PreInsertBefore);
5787 BasicBlock *Header =
5788 BasicBlock::Create(Context&: Ctx, Name: "omp_" + Name + ".header", Parent: F, InsertBefore: PreInsertBefore);
5789 BasicBlock *Cond =
5790 BasicBlock::Create(Context&: Ctx, Name: "omp_" + Name + ".cond", Parent: F, InsertBefore: PreInsertBefore);
5791 BasicBlock *Body =
5792 BasicBlock::Create(Context&: Ctx, Name: "omp_" + Name + ".body", Parent: F, InsertBefore: PreInsertBefore);
5793 BasicBlock *Latch =
5794 BasicBlock::Create(Context&: Ctx, Name: "omp_" + Name + ".inc", Parent: F, InsertBefore: PostInsertBefore);
5795 BasicBlock *Exit =
5796 BasicBlock::Create(Context&: Ctx, Name: "omp_" + Name + ".exit", Parent: F, InsertBefore: PostInsertBefore);
5797 BasicBlock *After =
5798 BasicBlock::Create(Context&: Ctx, Name: "omp_" + Name + ".after", Parent: F, InsertBefore: PostInsertBefore);
5799
5800 // Use specified DebugLoc for new instructions.
5801 Builder.SetCurrentDebugLocation(DL);
5802
5803 Builder.SetInsertPoint(Preheader);
5804 Builder.CreateBr(Dest: Header);
5805
5806 Builder.SetInsertPoint(Header);
5807 PHINode *IndVarPHI = Builder.CreatePHI(Ty: IndVarTy, NumReservedValues: 2, Name: "omp_" + Name + ".iv");
5808 IndVarPHI->addIncoming(V: ConstantInt::get(Ty: IndVarTy, V: 0), BB: Preheader);
5809 Builder.CreateBr(Dest: Cond);
5810
5811 Builder.SetInsertPoint(Cond);
5812 Value *Cmp =
5813 Builder.CreateICmpULT(LHS: IndVarPHI, RHS: TripCount, Name: "omp_" + Name + ".cmp");
5814 Builder.CreateCondBr(Cond: Cmp, True: Body, False: Exit);
5815
5816 Builder.SetInsertPoint(Body);
5817 Builder.CreateBr(Dest: Latch);
5818
5819 Builder.SetInsertPoint(Latch);
5820 // Decide whether the induction variable increment can carry nsw.
5821 //
5822 // Single loops: nsw is always kept (matching Clang). Any Fortran program
5823 // whose trip count overflows i32 is non-conforming per F2018 11.1.7.4.1, so
5824 // for valid programs 0 <= count <= INT_MAX always holds.
5825 //
5826 // Collapsed loops: the trip count is a product that can overflow i32 even for
5827 // a conforming program, so nsw is kept only when the product is a constant
5828 // that provably fits, dropped otherwise.
5829 bool HasNSW = Config.hasNoSignedWrap();
5830 if (HasNSW) {
5831 if (auto *CI = dyn_cast<ConstantInt>(Val: TripCount)) {
5832 unsigned BitWidth = CI->getType()->getIntegerBitWidth();
5833 APInt SignedMax = APInt::getSignedMaxValue(numBits: BitWidth);
5834 if (CI->getValue().ugt(RHS: SignedMax))
5835 HasNSW = false;
5836 } else if (IsCollapsed) {
5837 HasNSW = false;
5838 }
5839 }
5840 Value *Next =
5841 Builder.CreateAdd(LHS: IndVarPHI, RHS: ConstantInt::get(Ty: IndVarTy, V: 1),
5842 Name: "omp_" + Name + ".next", /*HasNUW=*/true, HasNSW);
5843 Builder.CreateBr(Dest: Header);
5844 IndVarPHI->addIncoming(V: Next, BB: Latch);
5845
5846 Builder.SetInsertPoint(Exit);
5847 Builder.CreateBr(Dest: After);
5848
5849 // Remember and return the canonical control flow.
5850 LoopInfos.emplace_front();
5851 CanonicalLoopInfo *CL = &LoopInfos.front();
5852
5853 CL->Header = Header;
5854 CL->Cond = Cond;
5855 CL->Latch = Latch;
5856 CL->Exit = Exit;
5857
5858#ifndef NDEBUG
5859 CL->assertOK();
5860#endif
5861 return CL;
5862}
5863
5864Expected<CanonicalLoopInfo *>
5865OpenMPIRBuilder::createCanonicalLoop(const LocationDescription &Loc,
5866 LoopBodyGenCallbackTy BodyGenCB,
5867 Value *TripCount, const Twine &Name) {
5868 BasicBlock *BB = Loc.IP.getBlock();
5869 BasicBlock *NextBB = BB->getNextNode();
5870
5871 CanonicalLoopInfo *CL = createLoopSkeleton(DL: Loc.DL, TripCount, F: BB->getParent(),
5872 PreInsertBefore: NextBB, PostInsertBefore: NextBB, Name);
5873 BasicBlock *After = CL->getAfter();
5874
5875 // If location is not set, don't connect the loop.
5876 if (updateToLocation(Loc)) {
5877 // Split the loop at the insertion point: Branch to the preheader and move
5878 // every following instruction to after the loop (the After BB). Also, the
5879 // new successor is the loop's after block.
5880 spliceBB(Builder, New: After, /*CreateBranch=*/false);
5881 Builder.CreateBr(Dest: CL->getPreheader());
5882 }
5883
5884 // Emit the body content. We do it after connecting the loop to the CFG to
5885 // avoid that the callback encounters degenerate BBs.
5886 if (Error Err = BodyGenCB(CL->getBodyIP(), CL->getIndVar()))
5887 return Err;
5888
5889#ifndef NDEBUG
5890 CL->assertOK();
5891#endif
5892 return CL;
5893}
5894
5895Expected<ScanInfo *> OpenMPIRBuilder::scanInfoInitialize() {
5896 ScanInfos.emplace_front();
5897 ScanInfo *Result = &ScanInfos.front();
5898 return Result;
5899}
5900
5901Expected<SmallVector<llvm::CanonicalLoopInfo *>>
5902OpenMPIRBuilder::createCanonicalScanLoops(
5903 const LocationDescription &Loc, LoopBodyGenCallbackTy BodyGenCB,
5904 Value *Start, Value *Stop, Value *Step, bool IsSigned, bool InclusiveStop,
5905 InsertPointTy ComputeIP, const Twine &Name, ScanInfo *ScanRedInfo) {
5906 LocationDescription ComputeLoc =
5907 ComputeIP.isSet() ? LocationDescription(ComputeIP, Loc.DL) : Loc;
5908 updateToLocation(Loc: ComputeLoc);
5909
5910 SmallVector<CanonicalLoopInfo *> Result;
5911
5912 Value *TripCount = calculateCanonicalLoopTripCount(
5913 Loc: ComputeLoc, Start, Stop, Step, IsSigned, InclusiveStop, Name);
5914 ScanRedInfo->Span = TripCount;
5915 ScanRedInfo->OMPScanInit = splitBB(Builder, CreateBranch: true, Name: "scan.init");
5916 Builder.SetInsertPoint(ScanRedInfo->OMPScanInit);
5917
5918 auto BodyGen = [=](InsertPointTy CodeGenIP, Value *IV) {
5919 Builder.restoreIP(IP: CodeGenIP);
5920 ScanRedInfo->IV = IV;
5921 createScanBBs(ScanRedInfo);
5922 BasicBlock *InputBlock = Builder.GetInsertBlock();
5923 Instruction *Terminator = InputBlock->getTerminator();
5924 assert(Terminator->getNumSuccessors() == 1);
5925 BasicBlock *ContinueBlock = Terminator->getSuccessor(Idx: 0);
5926 Terminator->setSuccessor(Idx: 0, BB: ScanRedInfo->OMPScanDispatch);
5927 emitBlock(BB: ScanRedInfo->OMPBeforeScanBlock,
5928 CurFn: Builder.GetInsertBlock()->getParent());
5929 Builder.CreateBr(Dest: ScanRedInfo->OMPScanLoopExit);
5930 emitBlock(BB: ScanRedInfo->OMPScanLoopExit,
5931 CurFn: Builder.GetInsertBlock()->getParent());
5932 Builder.CreateBr(Dest: ContinueBlock);
5933 Builder.SetInsertPoint(
5934 ScanRedInfo->OMPBeforeScanBlock->getFirstInsertionPt());
5935 return BodyGenCB(Builder.saveIP(), IV);
5936 };
5937
5938 const auto &&InputLoopGen = [&]() -> Error {
5939 Expected<CanonicalLoopInfo *> LoopInfo = createCanonicalLoop(
5940 Loc: Builder.saveIP(), BodyGenCB: BodyGen, Start, Stop, Step, IsSigned, InclusiveStop,
5941 ComputeIP, Name, InScan: true, ScanRedInfo);
5942 if (!LoopInfo)
5943 return LoopInfo.takeError();
5944 Result.push_back(Elt: *LoopInfo);
5945 Builder.restoreIP(IP: (*LoopInfo)->getAfterIP());
5946 return Error::success();
5947 };
5948 const auto &&ScanLoopGen = [&](LocationDescription Loc) -> Error {
5949 Expected<CanonicalLoopInfo *> LoopInfo =
5950 createCanonicalLoop(Loc, BodyGenCB: BodyGen, Start, Stop, Step, IsSigned,
5951 InclusiveStop, ComputeIP, Name, InScan: true, ScanRedInfo);
5952 if (!LoopInfo)
5953 return LoopInfo.takeError();
5954 Result.push_back(Elt: *LoopInfo);
5955 Builder.restoreIP(IP: (*LoopInfo)->getAfterIP());
5956 ScanRedInfo->OMPScanFinish = Builder.GetInsertBlock();
5957 return Error::success();
5958 };
5959 Error Err = emitScanBasedDirectiveIR(InputLoopGen, ScanLoopGen, ScanRedInfo);
5960 if (Err)
5961 return Err;
5962 return Result;
5963}
5964
5965Value *OpenMPIRBuilder::calculateCanonicalLoopTripCount(
5966 const LocationDescription &Loc, Value *Start, Value *Stop, Value *Step,
5967 bool IsSigned, bool InclusiveStop, const Twine &Name) {
5968
5969 // Consider the following difficulties (assuming 8-bit signed integers):
5970 // * Adding \p Step to the loop counter which passes \p Stop may overflow:
5971 // DO I = 1, 100, 50
5972 /// * A \p Step of INT_MIN cannot not be normalized to a positive direction:
5973 // DO I = 100, 0, -128
5974
5975 // Start, Stop and Step must be of the same integer type.
5976 auto *IndVarTy = cast<IntegerType>(Val: Start->getType());
5977 assert(IndVarTy == Stop->getType() && "Stop type mismatch");
5978 assert(IndVarTy == Step->getType() && "Step type mismatch");
5979
5980 updateToLocation(Loc);
5981
5982 ConstantInt *Zero = ConstantInt::get(Ty: IndVarTy, V: 0);
5983 ConstantInt *One = ConstantInt::get(Ty: IndVarTy, V: 1);
5984
5985 // Like Step, but always positive.
5986 Value *Incr = Step;
5987
5988 // Distance between Start and Stop; always positive.
5989 Value *Span;
5990
5991 // Condition whether there are no iterations are executed at all, e.g. because
5992 // UB < LB.
5993 Value *ZeroCmp;
5994
5995 if (IsSigned) {
5996 // Ensure that increment is positive. If not, negate and invert LB and UB.
5997 Value *IsNeg = Builder.CreateICmpSLT(LHS: Step, RHS: Zero);
5998 Incr = Builder.CreateSelect(C: IsNeg, True: Builder.CreateNeg(V: Step), False: Step);
5999 Value *LB = Builder.CreateSelect(C: IsNeg, True: Stop, False: Start);
6000 Value *UB = Builder.CreateSelect(C: IsNeg, True: Start, False: Stop);
6001 Span = Builder.CreateSub(LHS: UB, RHS: LB, Name: "", HasNUW: false, HasNSW: true);
6002 ZeroCmp = Builder.CreateICmp(
6003 P: InclusiveStop ? CmpInst::ICMP_SLT : CmpInst::ICMP_SLE, LHS: UB, RHS: LB);
6004 } else {
6005 Span = Builder.CreateSub(LHS: Stop, RHS: Start, Name: "", HasNUW: true);
6006 ZeroCmp = Builder.CreateICmp(
6007 P: InclusiveStop ? CmpInst::ICMP_ULT : CmpInst::ICMP_ULE, LHS: Stop, RHS: Start);
6008 }
6009
6010 Value *CountIfLooping;
6011 if (InclusiveStop) {
6012 CountIfLooping = Builder.CreateAdd(LHS: Builder.CreateUDiv(LHS: Span, RHS: Incr), RHS: One);
6013 } else {
6014 // Avoid incrementing past stop since it could overflow.
6015 Value *CountIfTwo = Builder.CreateAdd(
6016 LHS: Builder.CreateUDiv(LHS: Builder.CreateSub(LHS: Span, RHS: One), RHS: Incr), RHS: One);
6017 Value *OneCmp = Builder.CreateICmp(P: CmpInst::ICMP_ULE, LHS: Span, RHS: Incr);
6018 CountIfLooping = Builder.CreateSelect(C: OneCmp, True: One, False: CountIfTwo);
6019 }
6020
6021 return Builder.CreateSelect(C: ZeroCmp, True: Zero, False: CountIfLooping,
6022 Name: "omp_" + Name + ".tripcount");
6023}
6024
6025Expected<CanonicalLoopInfo *> OpenMPIRBuilder::createCanonicalLoop(
6026 const LocationDescription &Loc, LoopBodyGenCallbackTy BodyGenCB,
6027 Value *Start, Value *Stop, Value *Step, bool IsSigned, bool InclusiveStop,
6028 InsertPointTy ComputeIP, const Twine &Name, bool InScan,
6029 ScanInfo *ScanRedInfo) {
6030 LocationDescription ComputeLoc =
6031 ComputeIP.isSet() ? LocationDescription(ComputeIP, Loc.DL) : Loc;
6032
6033 Value *TripCount = calculateCanonicalLoopTripCount(
6034 Loc: ComputeLoc, Start, Stop, Step, IsSigned, InclusiveStop, Name);
6035
6036 auto BodyGen = [=](InsertPointTy CodeGenIP, Value *IV) {
6037 Builder.restoreIP(IP: CodeGenIP);
6038 Value *Span = Builder.CreateMul(LHS: IV, RHS: Step, Name: "", /*HasNUW=*/false,
6039 /*HasNSW=*/Config.hasNoSignedWrap());
6040 Value *IndVar = Builder.CreateAdd(LHS: Span, RHS: Start, Name: "", /*HasNUW=*/false,
6041 /*HasNSW=*/Config.hasNoSignedWrap());
6042 if (InScan)
6043 ScanRedInfo->IV = IndVar;
6044 return BodyGenCB(Builder.saveIP(), IndVar);
6045 };
6046 LocationDescription LoopLoc =
6047 ComputeIP.isSet()
6048 ? Loc
6049 : LocationDescription(Builder.saveIP(),
6050 Builder.getCurrentDebugLocation());
6051 return createCanonicalLoop(Loc: LoopLoc, BodyGenCB: BodyGen, TripCount, Name);
6052}
6053
6054// Returns an LLVM function to call for initializing loop bounds using OpenMP
6055// static scheduling for composite `distribute parallel for` depending on
6056// `type`. Only i32 and i64 are supported by the runtime. Always interpret
6057// integers as unsigned similarly to CanonicalLoopInfo.
6058static FunctionCallee
6059getKmpcDistForStaticInitForType(Type *Ty, Module &M,
6060 OpenMPIRBuilder &OMPBuilder) {
6061 unsigned Bitwidth = Ty->getIntegerBitWidth();
6062 if (Bitwidth == 32)
6063 return OMPBuilder.getOrCreateRuntimeFunction(
6064 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dist_for_static_init_4u);
6065 if (Bitwidth == 64)
6066 return OMPBuilder.getOrCreateRuntimeFunction(
6067 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dist_for_static_init_8u);
6068 llvm_unreachable("unknown OpenMP loop iterator bitwidth");
6069}
6070
6071// Returns an LLVM function to call for initializing loop bounds using OpenMP
6072// static scheduling depending on `type`. Only i32 and i64 are supported by the
6073// runtime. Always interpret integers as unsigned similarly to
6074// CanonicalLoopInfo.
6075static FunctionCallee getKmpcForStaticInitForType(Type *Ty, Module &M,
6076 OpenMPIRBuilder &OMPBuilder) {
6077 unsigned Bitwidth = Ty->getIntegerBitWidth();
6078 if (Bitwidth == 32)
6079 return OMPBuilder.getOrCreateRuntimeFunction(
6080 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_4u);
6081 if (Bitwidth == 64)
6082 return OMPBuilder.getOrCreateRuntimeFunction(
6083 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_8u);
6084 llvm_unreachable("unknown OpenMP loop iterator bitwidth");
6085}
6086
6087OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::applyStaticWorkshareLoop(
6088 DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP,
6089 WorksharingLoopType LoopType, bool NeedsBarrier, bool HasDistSchedule,
6090 OMPScheduleType DistScheduleSchedType) {
6091 assert(CLI->isValid() && "Requires a valid canonical loop");
6092 assert(!isConflictIP(AllocaIP, CLI->getPreheaderIP()) &&
6093 "Require dedicated allocate IP");
6094
6095 // Set up the source location value for OpenMP runtime.
6096 Builder.restoreIP(IP: CLI->getPreheaderIP());
6097 Builder.SetCurrentDebugLocation(DL);
6098
6099 uint32_t SrcLocStrSize;
6100 Constant *SrcLocStr = getOrCreateSrcLocStr(DL, SrcLocStrSize);
6101 IdentFlag Flag = IdentFlag(0);
6102 switch (LoopType) {
6103 case WorksharingLoopType::ForStaticLoop:
6104 Flag = OMP_IDENT_FLAG_WORK_LOOP;
6105 break;
6106 case WorksharingLoopType::DistributeStaticLoop:
6107 Flag = OMP_IDENT_FLAG_WORK_DISTRIBUTE;
6108 break;
6109 case WorksharingLoopType::DistributeForStaticLoop:
6110 Flag = OMP_IDENT_FLAG_WORK_DISTRIBUTE | OMP_IDENT_FLAG_WORK_LOOP;
6111 break;
6112 }
6113 Value *SrcLoc = getOrCreateIdent(SrcLocStr, SrcLocStrSize, LocFlags: Flag);
6114
6115 // Declare useful OpenMP runtime functions.
6116 Value *IV = CLI->getIndVar();
6117 Type *IVTy = IV->getType();
6118 FunctionCallee StaticInit =
6119 LoopType == WorksharingLoopType::DistributeForStaticLoop
6120 ? getKmpcDistForStaticInitForType(Ty: IVTy, M, OMPBuilder&: *this)
6121 : getKmpcForStaticInitForType(Ty: IVTy, M, OMPBuilder&: *this);
6122 FunctionCallee StaticFini =
6123 getOrCreateRuntimeFunction(M, FnID: omp::OMPRTL___kmpc_for_static_fini);
6124
6125 // Allocate space for computed loop bounds as expected by the "init" function.
6126 Builder.SetInsertPoint(AllocaIP.getBlock()->getFirstNonPHIOrDbgOrAlloca());
6127
6128 Type *I32Type = Type::getInt32Ty(C&: M.getContext());
6129 Value *PLastIter = Builder.CreateAlloca(Ty: I32Type, ArraySize: nullptr, Name: "p.lastiter");
6130 Value *PLowerBound = Builder.CreateAlloca(Ty: IVTy, ArraySize: nullptr, Name: "p.lowerbound");
6131 Value *PUpperBound = Builder.CreateAlloca(Ty: IVTy, ArraySize: nullptr, Name: "p.upperbound");
6132 Value *PStride = Builder.CreateAlloca(Ty: IVTy, ArraySize: nullptr, Name: "p.stride");
6133 CLI->setLastIter(PLastIter);
6134
6135 // At the end of the preheader, prepare for calling the "init" function by
6136 // storing the current loop bounds into the allocated space. A canonical loop
6137 // always iterates from 0 to trip-count with step 1. Note that "init" expects
6138 // and produces an inclusive upper bound.
6139 Builder.SetInsertPoint(CLI->getPreheader()->getTerminator());
6140 Constant *Zero = ConstantInt::get(Ty: IVTy, V: 0);
6141 Constant *One = ConstantInt::get(Ty: IVTy, V: 1);
6142 Builder.CreateStore(Val: Zero, Ptr: PLowerBound);
6143 Value *UpperBound = Builder.CreateSub(LHS: CLI->getTripCount(), RHS: One);
6144 Builder.CreateStore(Val: UpperBound, Ptr: PUpperBound);
6145 Builder.CreateStore(Val: One, Ptr: PStride);
6146
6147 Value *ThreadNum =
6148 getOrCreateThreadID(Ident: getOrCreateIdent(SrcLocStr, SrcLocStrSize));
6149
6150 OMPScheduleType SchedType =
6151 (LoopType == WorksharingLoopType::DistributeStaticLoop)
6152 ? OMPScheduleType::OrderedDistribute
6153 : OMPScheduleType::UnorderedStatic;
6154 Constant *SchedulingType =
6155 ConstantInt::get(Ty: I32Type, V: static_cast<int>(SchedType));
6156
6157 // Call the "init" function and update the trip count of the loop with the
6158 // value it produced.
6159 auto BuildInitCall = [LoopType, SrcLoc, ThreadNum, PLastIter, PLowerBound,
6160 PUpperBound, IVTy, PStride, One, Zero, StaticInit,
6161 this](Value *SchedulingType, auto &Builder) {
6162 SmallVector<Value *, 10> Args({SrcLoc, ThreadNum, SchedulingType, PLastIter,
6163 PLowerBound, PUpperBound});
6164 if (LoopType == WorksharingLoopType::DistributeForStaticLoop) {
6165 Value *PDistUpperBound =
6166 Builder.CreateAlloca(IVTy, nullptr, "p.distupperbound");
6167 Args.push_back(Elt: PDistUpperBound);
6168 }
6169 Args.append(IL: {PStride, One, Zero});
6170 createRuntimeFunctionCall(Callee: StaticInit, Args);
6171 };
6172 BuildInitCall(SchedulingType, Builder);
6173 if (HasDistSchedule &&
6174 LoopType != WorksharingLoopType::DistributeStaticLoop) {
6175 Constant *DistScheduleSchedType = ConstantInt::get(
6176 Ty: I32Type, V: static_cast<int>(omp::OMPScheduleType::OrderedDistribute));
6177 // We want to emit a second init function call for the dist_schedule clause
6178 // to the Distribute construct. This should only be done however if a
6179 // Workshare Loop is nested within a Distribute Construct
6180 BuildInitCall(DistScheduleSchedType, Builder);
6181 }
6182 Value *LowerBound = Builder.CreateLoad(Ty: IVTy, Ptr: PLowerBound);
6183 Value *InclusiveUpperBound = Builder.CreateLoad(Ty: IVTy, Ptr: PUpperBound);
6184 Value *TripCountMinusOne = Builder.CreateSub(LHS: InclusiveUpperBound, RHS: LowerBound);
6185 Value *TripCount = Builder.CreateAdd(LHS: TripCountMinusOne, RHS: One);
6186 CLI->setTripCount(TripCount);
6187
6188 // Update all uses of the induction variable except the one in the condition
6189 // block that compares it with the actual upper bound, and the increment in
6190 // the latch block.
6191
6192 CLI->mapIndVar(Updater: [&](Instruction *OldIV) -> Value * {
6193 Builder.SetInsertPoint(TheBB: CLI->getBody(),
6194 IP: CLI->getBody()->getFirstInsertionPt());
6195 Builder.SetCurrentDebugLocation(DL);
6196 return Builder.CreateAdd(LHS: OldIV, RHS: LowerBound, Name: "", /*HasNUW=*/false,
6197 /*HasNSW=*/Config.hasNoSignedWrap());
6198 });
6199
6200 // In the "exit" block, call the "fini" function.
6201 Builder.SetInsertPoint(TheBB: CLI->getExit(),
6202 IP: CLI->getExit()->getTerminator()->getIterator());
6203 createRuntimeFunctionCall(Callee: StaticFini, Args: {SrcLoc, ThreadNum});
6204
6205 // Add the barrier if requested.
6206 if (NeedsBarrier) {
6207 InsertPointOrErrorTy BarrierIP =
6208 createBarrier(Loc: LocationDescription(Builder.saveIP(), DL),
6209 Kind: omp::Directive::OMPD_for, /* ForceSimpleCall */ false,
6210 /* CheckCancelFlag */ false);
6211 if (!BarrierIP)
6212 return BarrierIP.takeError();
6213 }
6214
6215 InsertPointTy AfterIP = CLI->getAfterIP();
6216 CLI->invalidate();
6217
6218 return AfterIP;
6219}
6220
6221static void addAccessGroupMetadata(BasicBlock *Block, MDNode *AccessGroup,
6222 LoopInfo &LI);
6223static void addLoopMetadata(CanonicalLoopInfo *Loop,
6224 ArrayRef<Metadata *> Properties);
6225
6226static void applyParallelAccessesMetadata(CanonicalLoopInfo *CLI,
6227 LLVMContext &Ctx, Loop *Loop,
6228 LoopInfo &LoopInfo,
6229 SmallVector<Metadata *> &LoopMDList) {
6230 SmallSet<BasicBlock *, 8> Reachable;
6231
6232 // Get the basic blocks from the loop in which memref instructions
6233 // can be found.
6234 // TODO: Generalize getting all blocks inside a CanonicalizeLoopInfo,
6235 // preferably without running any passes.
6236 for (BasicBlock *Block : Loop->getBlocks()) {
6237 if (Block == CLI->getCond() || Block == CLI->getHeader())
6238 continue;
6239 Reachable.insert(Ptr: Block);
6240 }
6241
6242 // Add access group metadata to memory-access instructions.
6243 MDNode *AccessGroup = MDNode::getDistinct(Context&: Ctx, MDs: {});
6244 for (BasicBlock *BB : Reachable)
6245 addAccessGroupMetadata(Block: BB, AccessGroup, LI&: LoopInfo);
6246 // TODO: If the loop has existing parallel access metadata, have
6247 // to combine two lists.
6248 LoopMDList.push_back(Elt: MDNode::get(
6249 Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.parallel_accesses"), AccessGroup}));
6250}
6251
6252OpenMPIRBuilder::InsertPointOrErrorTy
6253OpenMPIRBuilder::applyStaticChunkedWorkshareLoop(
6254 DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP,
6255 bool NeedsBarrier, Value *ChunkSize, OMPScheduleType SchedType,
6256 Value *DistScheduleChunkSize, OMPScheduleType DistScheduleSchedType) {
6257 assert(CLI->isValid() && "Requires a valid canonical loop");
6258 assert((ChunkSize || DistScheduleChunkSize) && "Chunk size is required");
6259
6260 LLVMContext &Ctx = CLI->getFunction()->getContext();
6261 Value *IV = CLI->getIndVar();
6262 Value *OrigTripCount = CLI->getTripCount();
6263 Type *IVTy = IV->getType();
6264 assert(IVTy->getIntegerBitWidth() <= 64 &&
6265 "Max supported tripcount bitwidth is 64 bits");
6266 Type *InternalIVTy = IVTy->getIntegerBitWidth() <= 32 ? Type::getInt32Ty(C&: Ctx)
6267 : Type::getInt64Ty(C&: Ctx);
6268 Type *I32Type = Type::getInt32Ty(C&: M.getContext());
6269 Constant *Zero = ConstantInt::get(Ty: InternalIVTy, V: 0);
6270 Constant *One = ConstantInt::get(Ty: InternalIVTy, V: 1);
6271
6272 Function *F = CLI->getFunction();
6273 // Blocks must have terminators.
6274 // FIXME: Don't run analyses on incomplete/invalid IR.
6275 SmallVector<Instruction *> UIs;
6276 for (BasicBlock &BB : *F)
6277 if (!BB.hasTerminator())
6278 UIs.push_back(Elt: new UnreachableInst(F->getContext(), &BB));
6279 FunctionAnalysisManager FAM;
6280 FAM.registerPass(PassBuilder: []() { return DominatorTreeAnalysis(); });
6281 FAM.registerPass(PassBuilder: []() { return PassInstrumentationAnalysis(); });
6282 LoopAnalysis LIA;
6283 LoopInfo &&LI = LIA.run(F&: *F, AM&: FAM);
6284 for (Instruction *I : UIs)
6285 I->eraseFromParent();
6286 Loop *L = LI.getLoopFor(BB: CLI->getHeader());
6287 SmallVector<Metadata *> LoopMDList;
6288 if (ChunkSize || DistScheduleChunkSize)
6289 applyParallelAccessesMetadata(CLI, Ctx, Loop: L, LoopInfo&: LI, LoopMDList);
6290 addLoopMetadata(Loop: CLI, Properties: LoopMDList);
6291
6292 // Declare useful OpenMP runtime functions.
6293 FunctionCallee StaticInit =
6294 getKmpcForStaticInitForType(Ty: InternalIVTy, M, OMPBuilder&: *this);
6295 FunctionCallee StaticFini =
6296 getOrCreateRuntimeFunction(M, FnID: omp::OMPRTL___kmpc_for_static_fini);
6297
6298 // Allocate space for computed loop bounds as expected by the "init" function.
6299 Builder.restoreIP(IP: AllocaIP);
6300 Builder.SetCurrentDebugLocation(DL);
6301 Value *PLastIter = Builder.CreateAlloca(Ty: I32Type, ArraySize: nullptr, Name: "p.lastiter");
6302 Value *PLowerBound =
6303 Builder.CreateAlloca(Ty: InternalIVTy, ArraySize: nullptr, Name: "p.lowerbound");
6304 Value *PUpperBound =
6305 Builder.CreateAlloca(Ty: InternalIVTy, ArraySize: nullptr, Name: "p.upperbound");
6306 Value *PStride = Builder.CreateAlloca(Ty: InternalIVTy, ArraySize: nullptr, Name: "p.stride");
6307 CLI->setLastIter(PLastIter);
6308
6309 // Set up the source location value for the OpenMP runtime.
6310 Builder.restoreIP(IP: CLI->getPreheaderIP());
6311 Builder.SetCurrentDebugLocation(DL);
6312
6313 // TODO: Detect overflow in ubsan or max-out with current tripcount.
6314 Value *CastedChunkSize = Builder.CreateZExtOrTrunc(
6315 V: ChunkSize ? ChunkSize : Zero, DestTy: InternalIVTy, Name: "chunksize");
6316 Value *CastedDistScheduleChunkSize = Builder.CreateZExtOrTrunc(
6317 V: DistScheduleChunkSize ? DistScheduleChunkSize : Zero, DestTy: InternalIVTy,
6318 Name: "distschedulechunksize");
6319 Value *CastedTripCount =
6320 Builder.CreateZExt(V: OrigTripCount, DestTy: InternalIVTy, Name: "tripcount");
6321
6322 Constant *SchedulingType =
6323 ConstantInt::get(Ty: I32Type, V: static_cast<int>(SchedType));
6324 Constant *DistSchedulingType =
6325 ConstantInt::get(Ty: I32Type, V: static_cast<int>(DistScheduleSchedType));
6326 Builder.CreateStore(Val: Zero, Ptr: PLowerBound);
6327 Value *OrigUpperBound = Builder.CreateSub(LHS: CastedTripCount, RHS: One);
6328 Value *IsTripCountZero = Builder.CreateICmpEQ(LHS: CastedTripCount, RHS: Zero);
6329 Value *UpperBound =
6330 Builder.CreateSelect(C: IsTripCountZero, True: Zero, False: OrigUpperBound);
6331 Builder.CreateStore(Val: UpperBound, Ptr: PUpperBound);
6332 Builder.CreateStore(Val: One, Ptr: PStride);
6333
6334 // Call the "init" function and update the trip count of the loop with the
6335 // value it produced.
6336 uint32_t SrcLocStrSize;
6337 Constant *SrcLocStr = getOrCreateSrcLocStr(DL, SrcLocStrSize);
6338 IdentFlag Flag = OMP_IDENT_FLAG_WORK_LOOP;
6339 if (DistScheduleSchedType != OMPScheduleType::None) {
6340 Flag |= OMP_IDENT_FLAG_WORK_DISTRIBUTE;
6341 }
6342 Value *SrcLoc = getOrCreateIdent(SrcLocStr, SrcLocStrSize, LocFlags: Flag);
6343 Value *ThreadNum =
6344 getOrCreateThreadID(Ident: getOrCreateIdent(SrcLocStr, SrcLocStrSize));
6345 auto BuildInitCall = [StaticInit, SrcLoc, ThreadNum, PLastIter, PLowerBound,
6346 PUpperBound, PStride, One,
6347 this](Value *SchedulingType, Value *ChunkSize,
6348 auto &Builder) {
6349 createRuntimeFunctionCall(
6350 Callee: StaticInit, Args: {/*loc=*/SrcLoc, /*global_tid=*/ThreadNum,
6351 /*schedtype=*/SchedulingType, /*plastiter=*/PLastIter,
6352 /*plower=*/PLowerBound, /*pupper=*/PUpperBound,
6353 /*pstride=*/PStride, /*incr=*/One,
6354 /*chunk=*/ChunkSize});
6355 };
6356 BuildInitCall(SchedulingType, CastedChunkSize, Builder);
6357 if (DistScheduleSchedType != OMPScheduleType::None &&
6358 SchedType != OMPScheduleType::OrderedDistributeChunked &&
6359 SchedType != OMPScheduleType::OrderedDistribute) {
6360 // We want to emit a second init function call for the dist_schedule clause
6361 // to the Distribute construct. This should only be done however if a
6362 // Workshare Loop is nested within a Distribute Construct
6363 BuildInitCall(DistSchedulingType, CastedDistScheduleChunkSize, Builder);
6364 }
6365
6366 // Load values written by the "init" function.
6367 Value *FirstChunkStart =
6368 Builder.CreateLoad(Ty: InternalIVTy, Ptr: PLowerBound, Name: "omp_firstchunk.lb");
6369 Value *FirstChunkStop =
6370 Builder.CreateLoad(Ty: InternalIVTy, Ptr: PUpperBound, Name: "omp_firstchunk.ub");
6371 Value *FirstChunkEnd = Builder.CreateAdd(LHS: FirstChunkStop, RHS: One);
6372 Value *ChunkRange =
6373 Builder.CreateSub(LHS: FirstChunkEnd, RHS: FirstChunkStart, Name: "omp_chunk.range");
6374 Value *NextChunkStride =
6375 Builder.CreateLoad(Ty: InternalIVTy, Ptr: PStride, Name: "omp_dispatch.stride");
6376
6377 // Create outer "dispatch" loop for enumerating the chunks.
6378 BasicBlock *DispatchEnter = splitBB(Builder, CreateBranch: true);
6379 Value *DispatchCounter;
6380
6381 // It is safe to assume this didn't return an error because the callback
6382 // passed into createCanonicalLoop is the only possible error source, and it
6383 // always returns success.
6384 CanonicalLoopInfo *DispatchCLI = cantFail(ValOrErr: createCanonicalLoop(
6385 Loc: {Builder.saveIP(), DL},
6386 BodyGenCB: [&](InsertPointTy BodyIP, Value *Counter) {
6387 DispatchCounter = Counter;
6388 return Error::success();
6389 },
6390 Start: FirstChunkStart, Stop: CastedTripCount, Step: NextChunkStride,
6391 /*IsSigned=*/false, /*InclusiveStop=*/false, /*ComputeIP=*/{},
6392 Name: "dispatch"));
6393
6394 // Remember the BasicBlocks of the dispatch loop we need, then invalidate to
6395 // not have to preserve the canonical invariant.
6396 BasicBlock *DispatchBody = DispatchCLI->getBody();
6397 BasicBlock *DispatchLatch = DispatchCLI->getLatch();
6398 BasicBlock *DispatchExit = DispatchCLI->getExit();
6399 BasicBlock *DispatchAfter = DispatchCLI->getAfter();
6400 DispatchCLI->invalidate();
6401
6402 // Rewire the original loop to become the chunk loop inside the dispatch loop.
6403 redirectTo(Source: DispatchAfter, Target: CLI->getAfter(), DL);
6404 redirectTo(Source: CLI->getExit(), Target: DispatchLatch, DL);
6405 redirectTo(Source: DispatchBody, Target: DispatchEnter, DL);
6406
6407 // Prepare the prolog of the chunk loop.
6408 Builder.restoreIP(IP: CLI->getPreheaderIP());
6409 Builder.SetCurrentDebugLocation(DL);
6410
6411 // Compute the number of iterations of the chunk loop.
6412 Builder.SetInsertPoint(CLI->getPreheader()->getTerminator());
6413 Value *ChunkEnd = Builder.CreateAdd(LHS: DispatchCounter, RHS: ChunkRange);
6414 Value *IsLastChunk =
6415 Builder.CreateICmpUGE(LHS: ChunkEnd, RHS: CastedTripCount, Name: "omp_chunk.is_last");
6416 Value *CountUntilOrigTripCount =
6417 Builder.CreateSub(LHS: CastedTripCount, RHS: DispatchCounter);
6418 Value *ChunkTripCount = Builder.CreateSelect(
6419 C: IsLastChunk, True: CountUntilOrigTripCount, False: ChunkRange, Name: "omp_chunk.tripcount");
6420 Value *BackcastedChunkTC =
6421 Builder.CreateTrunc(V: ChunkTripCount, DestTy: IVTy, Name: "omp_chunk.tripcount.trunc");
6422 CLI->setTripCount(BackcastedChunkTC);
6423
6424 // Update all uses of the induction variable except the one in the condition
6425 // block that compares it with the actual upper bound, and the increment in
6426 // the latch block.
6427 Value *BackcastedDispatchCounter =
6428 Builder.CreateTrunc(V: DispatchCounter, DestTy: IVTy, Name: "omp_dispatch.iv.trunc");
6429 CLI->mapIndVar(Updater: [&](Instruction *) -> Value * {
6430 Builder.restoreIP(IP: CLI->getBodyIP());
6431 return Builder.CreateAdd(LHS: IV, RHS: BackcastedDispatchCounter);
6432 });
6433
6434 // In the "exit" block, call the "fini" function.
6435 Builder.SetInsertPoint(TheBB: DispatchExit, IP: DispatchExit->getFirstInsertionPt());
6436 createRuntimeFunctionCall(Callee: StaticFini, Args: {SrcLoc, ThreadNum});
6437
6438 // Add the barrier if requested.
6439 if (NeedsBarrier) {
6440 InsertPointOrErrorTy AfterIP =
6441 createBarrier(Loc: LocationDescription(Builder.saveIP(), DL), Kind: OMPD_for,
6442 /*ForceSimpleCall=*/false, /*CheckCancelFlag=*/false);
6443 if (!AfterIP)
6444 return AfterIP.takeError();
6445 }
6446
6447#ifndef NDEBUG
6448 // Even though we currently do not support applying additional methods to it,
6449 // the chunk loop should remain a canonical loop.
6450 CLI->assertOK();
6451#endif
6452
6453 return InsertPointTy(DispatchAfter, DispatchAfter->getFirstInsertionPt());
6454}
6455
6456// Returns an LLVM function to call for executing an OpenMP static worksharing
6457// for loop depending on `type`. Only i32 and i64 are supported by the runtime.
6458// Always interpret integers as unsigned similarly to CanonicalLoopInfo.
6459static FunctionCallee
6460getKmpcForStaticLoopForType(Type *Ty, OpenMPIRBuilder *OMPBuilder,
6461 WorksharingLoopType LoopType) {
6462 unsigned Bitwidth = Ty->getIntegerBitWidth();
6463 Module &M = OMPBuilder->M;
6464 switch (LoopType) {
6465 case WorksharingLoopType::ForStaticLoop:
6466 if (Bitwidth == 32)
6467 return OMPBuilder->getOrCreateRuntimeFunction(
6468 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_for_static_loop_4u);
6469 if (Bitwidth == 64)
6470 return OMPBuilder->getOrCreateRuntimeFunction(
6471 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_for_static_loop_8u);
6472 break;
6473 case WorksharingLoopType::DistributeStaticLoop:
6474 if (Bitwidth == 32)
6475 return OMPBuilder->getOrCreateRuntimeFunction(
6476 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_distribute_static_loop_4u);
6477 if (Bitwidth == 64)
6478 return OMPBuilder->getOrCreateRuntimeFunction(
6479 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_distribute_static_loop_8u);
6480 break;
6481 case WorksharingLoopType::DistributeForStaticLoop:
6482 if (Bitwidth == 32)
6483 return OMPBuilder->getOrCreateRuntimeFunction(
6484 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_distribute_for_static_loop_4u);
6485 if (Bitwidth == 64)
6486 return OMPBuilder->getOrCreateRuntimeFunction(
6487 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_distribute_for_static_loop_8u);
6488 break;
6489 }
6490 if (Bitwidth != 32 && Bitwidth != 64) {
6491 llvm_unreachable("Unknown OpenMP loop iterator bitwidth");
6492 }
6493 llvm_unreachable("Unknown type of OpenMP worksharing loop");
6494}
6495
6496// Inserts a call to proper OpenMP Device RTL function which handles
6497// loop worksharing.
6498static void createTargetLoopWorkshareCall(OpenMPIRBuilder *OMPBuilder,
6499 WorksharingLoopType LoopType,
6500 BasicBlock *InsertBlock, Value *Ident,
6501 Value *LoopBodyArg, Value *TripCount,
6502 Function &LoopBodyFn, bool NoLoop) {
6503 Type *TripCountTy = TripCount->getType();
6504 Module &M = OMPBuilder->M;
6505 IRBuilder<> &Builder = OMPBuilder->Builder;
6506 FunctionCallee RTLFn =
6507 getKmpcForStaticLoopForType(Ty: TripCountTy, OMPBuilder, LoopType);
6508 SmallVector<Value *, 8> RealArgs;
6509 RealArgs.push_back(Elt: Ident);
6510 RealArgs.push_back(Elt: &LoopBodyFn);
6511 RealArgs.push_back(Elt: LoopBodyArg);
6512 RealArgs.push_back(Elt: TripCount);
6513 if (LoopType == WorksharingLoopType::DistributeStaticLoop) {
6514 RealArgs.push_back(Elt: ConstantInt::get(Ty: TripCountTy, V: 0));
6515 RealArgs.push_back(Elt: ConstantInt::get(Ty: Builder.getInt8Ty(), V: 0));
6516 Builder.restoreIP(IP: {InsertBlock, std::prev(x: InsertBlock->end())});
6517 OMPBuilder->createRuntimeFunctionCall(Callee: RTLFn, Args: RealArgs);
6518 return;
6519 }
6520 FunctionCallee RTLNumThreads = OMPBuilder->getOrCreateRuntimeFunction(
6521 M, FnID: omp::RuntimeFunction::OMPRTL_omp_get_num_threads);
6522 Builder.restoreIP(IP: {InsertBlock, std::prev(x: InsertBlock->end())});
6523 Value *NumThreads = OMPBuilder->createRuntimeFunctionCall(Callee: RTLNumThreads, Args: {});
6524
6525 RealArgs.push_back(
6526 Elt: Builder.CreateZExtOrTrunc(V: NumThreads, DestTy: TripCountTy, Name: "num.threads.cast"));
6527 RealArgs.push_back(Elt: ConstantInt::get(Ty: TripCountTy, V: 0));
6528 if (LoopType == WorksharingLoopType::DistributeForStaticLoop) {
6529 RealArgs.push_back(Elt: ConstantInt::get(Ty: TripCountTy, V: 0));
6530 RealArgs.push_back(Elt: ConstantInt::get(Ty: Builder.getInt8Ty(), V: NoLoop));
6531 } else {
6532 RealArgs.push_back(Elt: ConstantInt::get(Ty: Builder.getInt8Ty(), V: 0));
6533 }
6534
6535 OMPBuilder->createRuntimeFunctionCall(Callee: RTLFn, Args: RealArgs);
6536}
6537
6538static void workshareLoopTargetCallback(
6539 OpenMPIRBuilder *OMPIRBuilder, CanonicalLoopInfo *CLI, Value *Ident,
6540 Function &OutlinedFn, const SmallVector<Instruction *, 4> &ToBeDeleted,
6541 WorksharingLoopType LoopType, bool NoLoop) {
6542 IRBuilder<> &Builder = OMPIRBuilder->Builder;
6543 BasicBlock *Preheader = CLI->getPreheader();
6544 Value *TripCount = CLI->getTripCount();
6545
6546 // After loop body outling, the loop body contains only set up
6547 // of loop body argument structure and the call to the outlined
6548 // loop body function. Firstly, we need to move setup of loop body args
6549 // into loop preheader.
6550 Preheader->splice(ToIt: std::prev(x: Preheader->end()), FromBB: CLI->getBody(),
6551 FromBeginIt: CLI->getBody()->begin(), FromEndIt: std::prev(x: CLI->getBody()->end()));
6552
6553 // The next step is to remove the whole loop. We do not it need anymore.
6554 // That's why make an unconditional branch from loop preheader to loop
6555 // exit block
6556 Builder.restoreIP(IP: {Preheader, Preheader->end()});
6557 Builder.SetCurrentDebugLocation(Preheader->getTerminator()->getDebugLoc());
6558 Preheader->getTerminator()->eraseFromParent();
6559 Builder.CreateBr(Dest: CLI->getExit());
6560
6561 // Delete dead loop blocks
6562 OpenMPIRBuilder::OutlineInfo CleanUpInfo;
6563 SmallPtrSet<BasicBlock *, 32> RegionBlockSet;
6564 SmallVector<BasicBlock *, 32> BlocksToBeRemoved;
6565 CleanUpInfo.EntryBB = CLI->getHeader();
6566 CleanUpInfo.ExitBB = CLI->getExit();
6567 CleanUpInfo.collectBlocks(BlockSet&: RegionBlockSet, BlockVector&: BlocksToBeRemoved);
6568 DeleteDeadBlocks(BBs: BlocksToBeRemoved);
6569
6570 // Find the instruction which corresponds to loop body argument structure
6571 // and remove the call to loop body function instruction.
6572 Value *LoopBodyArg;
6573 User *OutlinedFnUser = OutlinedFn.getUniqueUndroppableUser();
6574 assert(OutlinedFnUser &&
6575 "Expected unique undroppable user of outlined function");
6576 CallInst *OutlinedFnCallInstruction = dyn_cast<CallInst>(Val: OutlinedFnUser);
6577 assert(OutlinedFnCallInstruction && "Expected outlined function call");
6578 assert((OutlinedFnCallInstruction->getParent() == Preheader) &&
6579 "Expected outlined function call to be located in loop preheader");
6580 // Check in case no argument structure has been passed.
6581 if (OutlinedFnCallInstruction->arg_size() > 1)
6582 LoopBodyArg = OutlinedFnCallInstruction->getArgOperand(i: 1);
6583 else
6584 LoopBodyArg = Constant::getNullValue(Ty: Builder.getPtrTy());
6585 OutlinedFnCallInstruction->eraseFromParent();
6586
6587 createTargetLoopWorkshareCall(OMPBuilder: OMPIRBuilder, LoopType, InsertBlock: Preheader, Ident,
6588 LoopBodyArg, TripCount, LoopBodyFn&: OutlinedFn, NoLoop);
6589
6590 for (auto &ToBeDeletedItem : ToBeDeleted)
6591 ToBeDeletedItem->eraseFromParent();
6592 CLI->invalidate();
6593}
6594
6595OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::applyWorkshareLoopTarget(
6596 DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP,
6597 WorksharingLoopType LoopType, bool NoLoop) {
6598 uint32_t SrcLocStrSize;
6599 Constant *SrcLocStr = getOrCreateSrcLocStr(DL, SrcLocStrSize);
6600 IdentFlag Flag = IdentFlag(0);
6601 switch (LoopType) {
6602 case WorksharingLoopType::ForStaticLoop:
6603 Flag = OMP_IDENT_FLAG_WORK_LOOP;
6604 break;
6605 case WorksharingLoopType::DistributeStaticLoop:
6606 Flag = OMP_IDENT_FLAG_WORK_DISTRIBUTE;
6607 break;
6608 case WorksharingLoopType::DistributeForStaticLoop:
6609 Flag = OMP_IDENT_FLAG_WORK_DISTRIBUTE | OMP_IDENT_FLAG_WORK_LOOP;
6610 break;
6611 }
6612 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize, LocFlags: Flag);
6613
6614 auto OI = std::make_unique<OutlineInfo>();
6615 OI->OuterAllocBB = CLI->getPreheader();
6616 Function *OuterFn = CLI->getPreheader()->getParent();
6617
6618 // Instructions which need to be deleted at the end of code generation
6619 SmallVector<Instruction *, 4> ToBeDeleted;
6620
6621 OI->OuterAllocBB = AllocaIP.getBlock();
6622
6623 // Mark the body loop as region which needs to be extracted
6624 OI->EntryBB = CLI->getBody();
6625 OI->ExitBB = CLI->getLatch()->splitBasicBlockBefore(I: CLI->getLatch()->begin(),
6626 BBName: "omp.prelatch");
6627
6628 // Prepare loop body for extraction
6629 Builder.restoreIP(IP: {CLI->getPreheader(), CLI->getPreheader()->begin()});
6630
6631 // Insert new loop counter variable which will be used only in loop
6632 // body.
6633 AllocaInst *NewLoopCnt = Builder.CreateAlloca(Ty: CLI->getIndVarType(), ArraySize: 0, Name: "");
6634 Instruction *NewLoopCntLoad =
6635 Builder.CreateLoad(Ty: CLI->getIndVarType(), Ptr: NewLoopCnt);
6636 // New loop counter instructions are redundant in the loop preheader when
6637 // code generation for workshare loop is finshed. That's why mark them as
6638 // ready for deletion.
6639 ToBeDeleted.push_back(Elt: NewLoopCntLoad);
6640 ToBeDeleted.push_back(Elt: NewLoopCnt);
6641
6642 // Analyse loop body region. Find all input variables which are used inside
6643 // loop body region.
6644 SmallPtrSet<BasicBlock *, 32> ParallelRegionBlockSet;
6645 SmallVector<BasicBlock *, 32> Blocks;
6646 OI->collectBlocks(BlockSet&: ParallelRegionBlockSet, BlockVector&: Blocks);
6647
6648 CodeExtractorAnalysisCache CEAC(*OuterFn);
6649 CodeExtractor Extractor(Blocks,
6650 /* DominatorTree */ nullptr,
6651 /* AggregateArgs */ true,
6652 /* BlockFrequencyInfo */ nullptr,
6653 /* BranchProbabilityInfo */ nullptr,
6654 /* AssumptionCache */ nullptr,
6655 /* AllowVarArgs */ true,
6656 /* AllowAlloca */ true,
6657 /* AllocationBlock */ CLI->getPreheader(),
6658 /* DeallocationBlocks */ {},
6659 /* Suffix */ ".omp_wsloop",
6660 /* AggrArgsIn0AddrSpace */ true);
6661
6662 BasicBlock *CommonExit = nullptr;
6663 SetVector<Value *> SinkingCands, HoistingCands;
6664
6665 // Find allocas outside the loop body region which are used inside loop
6666 // body
6667 Extractor.findAllocas(CEAC, SinkCands&: SinkingCands, HoistCands&: HoistingCands, ExitBlock&: CommonExit);
6668
6669 // We need to model loop body region as the function f(cnt, loop_arg).
6670 // That's why we replace loop induction variable by the new counter
6671 // which will be one of loop body function argument
6672 SmallVector<User *> Users(CLI->getIndVar()->user_begin(),
6673 CLI->getIndVar()->user_end());
6674 for (auto Use : Users) {
6675 if (Instruction *Inst = dyn_cast<Instruction>(Val: Use)) {
6676 if (ParallelRegionBlockSet.count(Ptr: Inst->getParent())) {
6677 Inst->replaceUsesOfWith(From: CLI->getIndVar(), To: NewLoopCntLoad);
6678 }
6679 }
6680 }
6681 // Make sure that loop counter variable is not merged into loop body
6682 // function argument structure and it is passed as separate variable
6683 OI->ExcludeArgsFromAggregate.push_back(Elt: NewLoopCntLoad);
6684
6685 // PostOutline CB is invoked when loop body function is outlined and
6686 // loop body is replaced by call to outlined function. We need to add
6687 // call to OpenMP device rtl inside loop preheader. OpenMP device rtl
6688 // function will handle loop control logic.
6689 //
6690 OI->PostOutlineCB = [=, ToBeDeletedVec =
6691 std::move(ToBeDeleted)](Function &OutlinedFn) {
6692 workshareLoopTargetCallback(OMPIRBuilder: this, CLI, Ident, OutlinedFn, ToBeDeleted: ToBeDeletedVec,
6693 LoopType, NoLoop);
6694 };
6695 addOutlineInfo(OI: std::move(OI));
6696 return CLI->getAfterIP();
6697}
6698
6699OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::applyWorkshareLoop(
6700 DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP,
6701 bool NeedsBarrier, omp::ScheduleKind SchedKind, Value *ChunkSize,
6702 bool HasSimdModifier, bool HasMonotonicModifier,
6703 bool HasNonmonotonicModifier, bool HasOrderedClause,
6704 WorksharingLoopType LoopType, bool NoLoop, bool HasDistSchedule,
6705 Value *DistScheduleChunkSize) {
6706 if (Config.isTargetDevice())
6707 return applyWorkshareLoopTarget(DL, CLI, AllocaIP, LoopType, NoLoop);
6708 OMPScheduleType EffectiveScheduleType = computeOpenMPScheduleType(
6709 ClauseKind: SchedKind, HasChunks: ChunkSize, HasSimdModifier, HasMonotonicModifier,
6710 HasNonmonotonicModifier, HasOrderedClause, HasDistScheduleChunks: DistScheduleChunkSize);
6711
6712 bool IsOrdered = (EffectiveScheduleType & OMPScheduleType::ModifierOrdered) ==
6713 OMPScheduleType::ModifierOrdered;
6714 OMPScheduleType DistScheduleSchedType = OMPScheduleType::None;
6715 if (HasDistSchedule) {
6716 DistScheduleSchedType = DistScheduleChunkSize
6717 ? OMPScheduleType::OrderedDistributeChunked
6718 : OMPScheduleType::OrderedDistribute;
6719 }
6720 switch (EffectiveScheduleType & ~OMPScheduleType::ModifierMask) {
6721 case OMPScheduleType::BaseStatic:
6722 case OMPScheduleType::BaseDistribute:
6723 assert((!ChunkSize || !DistScheduleChunkSize) &&
6724 "No chunk size with static-chunked schedule");
6725 if (IsOrdered && !HasDistSchedule)
6726 return applyDynamicWorkshareLoop(DL, CLI, AllocaIP, SchedType: EffectiveScheduleType,
6727 NeedsBarrier, Chunk: ChunkSize);
6728 // FIXME: Monotonicity ignored?
6729 if (DistScheduleChunkSize)
6730 return applyStaticChunkedWorkshareLoop(
6731 DL, CLI, AllocaIP, NeedsBarrier, ChunkSize, SchedType: EffectiveScheduleType,
6732 DistScheduleChunkSize, DistScheduleSchedType);
6733 return applyStaticWorkshareLoop(DL, CLI, AllocaIP, LoopType, NeedsBarrier,
6734 HasDistSchedule);
6735
6736 case OMPScheduleType::BaseStaticChunked:
6737 case OMPScheduleType::BaseDistributeChunked:
6738 if (IsOrdered && !HasDistSchedule)
6739 return applyDynamicWorkshareLoop(DL, CLI, AllocaIP, SchedType: EffectiveScheduleType,
6740 NeedsBarrier, Chunk: ChunkSize);
6741 // FIXME: Monotonicity ignored?
6742 return applyStaticChunkedWorkshareLoop(
6743 DL, CLI, AllocaIP, NeedsBarrier, ChunkSize, SchedType: EffectiveScheduleType,
6744 DistScheduleChunkSize, DistScheduleSchedType);
6745
6746 case OMPScheduleType::BaseRuntime:
6747 case OMPScheduleType::BaseAuto:
6748 case OMPScheduleType::BaseGreedy:
6749 case OMPScheduleType::BaseBalanced:
6750 case OMPScheduleType::BaseSteal:
6751 case OMPScheduleType::BaseRuntimeSimd:
6752 assert(!ChunkSize &&
6753 "schedule type does not support user-defined chunk sizes");
6754 [[fallthrough]];
6755 case OMPScheduleType::BaseGuidedSimd:
6756 case OMPScheduleType::BaseDynamicChunked:
6757 case OMPScheduleType::BaseGuidedChunked:
6758 case OMPScheduleType::BaseGuidedIterativeChunked:
6759 case OMPScheduleType::BaseGuidedAnalyticalChunked:
6760 case OMPScheduleType::BaseStaticBalancedChunked:
6761 return applyDynamicWorkshareLoop(DL, CLI, AllocaIP, SchedType: EffectiveScheduleType,
6762 NeedsBarrier, Chunk: ChunkSize);
6763
6764 default:
6765 llvm_unreachable("Unknown/unimplemented schedule kind");
6766 }
6767}
6768
6769/// Returns an LLVM function to call for initializing loop bounds using OpenMP
6770/// dynamic scheduling depending on `type`. Only i32 and i64 are supported by
6771/// the runtime. Always interpret integers as unsigned similarly to
6772/// CanonicalLoopInfo.
6773static FunctionCallee
6774getKmpcForDynamicInitForType(Type *Ty, Module &M, OpenMPIRBuilder &OMPBuilder) {
6775 unsigned Bitwidth = Ty->getIntegerBitWidth();
6776 if (Bitwidth == 32)
6777 return OMPBuilder.getOrCreateRuntimeFunction(
6778 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dispatch_init_4u);
6779 if (Bitwidth == 64)
6780 return OMPBuilder.getOrCreateRuntimeFunction(
6781 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dispatch_init_8u);
6782 llvm_unreachable("unknown OpenMP loop iterator bitwidth");
6783}
6784
6785/// Returns an LLVM function to call for updating the next loop using OpenMP
6786/// dynamic scheduling depending on `type`. Only i32 and i64 are supported by
6787/// the runtime. Always interpret integers as unsigned similarly to
6788/// CanonicalLoopInfo.
6789static FunctionCallee
6790getKmpcForDynamicNextForType(Type *Ty, Module &M, OpenMPIRBuilder &OMPBuilder) {
6791 unsigned Bitwidth = Ty->getIntegerBitWidth();
6792 if (Bitwidth == 32)
6793 return OMPBuilder.getOrCreateRuntimeFunction(
6794 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dispatch_next_4u);
6795 if (Bitwidth == 64)
6796 return OMPBuilder.getOrCreateRuntimeFunction(
6797 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dispatch_next_8u);
6798 llvm_unreachable("unknown OpenMP loop iterator bitwidth");
6799}
6800
6801/// Returns an LLVM function to call for finalizing the dynamic loop using
6802/// depending on `type`. Only i32 and i64 are supported by the runtime. Always
6803/// interpret integers as unsigned similarly to CanonicalLoopInfo.
6804static FunctionCallee
6805getKmpcForDynamicFiniForType(Type *Ty, Module &M, OpenMPIRBuilder &OMPBuilder) {
6806 unsigned Bitwidth = Ty->getIntegerBitWidth();
6807 if (Bitwidth == 32)
6808 return OMPBuilder.getOrCreateRuntimeFunction(
6809 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dispatch_fini_4u);
6810 if (Bitwidth == 64)
6811 return OMPBuilder.getOrCreateRuntimeFunction(
6812 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dispatch_fini_8u);
6813 llvm_unreachable("unknown OpenMP loop iterator bitwidth");
6814}
6815
6816OpenMPIRBuilder::InsertPointOrErrorTy
6817OpenMPIRBuilder::applyDynamicWorkshareLoop(DebugLoc DL, CanonicalLoopInfo *CLI,
6818 InsertPointTy AllocaIP,
6819 OMPScheduleType SchedType,
6820 bool NeedsBarrier, Value *Chunk) {
6821 assert(CLI->isValid() && "Requires a valid canonical loop");
6822 assert(!isConflictIP(AllocaIP, CLI->getPreheaderIP()) &&
6823 "Require dedicated allocate IP");
6824 assert(isValidWorkshareLoopScheduleType(SchedType) &&
6825 "Require valid schedule type");
6826
6827 bool Ordered = (SchedType & OMPScheduleType::ModifierOrdered) ==
6828 OMPScheduleType::ModifierOrdered;
6829
6830 // Set up the source location value for OpenMP runtime.
6831 Builder.SetCurrentDebugLocation(DL);
6832
6833 uint32_t SrcLocStrSize;
6834 Constant *SrcLocStr = getOrCreateSrcLocStr(DL, SrcLocStrSize);
6835 Value *SrcLoc =
6836 getOrCreateIdent(SrcLocStr, SrcLocStrSize, LocFlags: OMP_IDENT_FLAG_WORK_LOOP);
6837
6838 // Declare useful OpenMP runtime functions.
6839 Value *IV = CLI->getIndVar();
6840 Type *IVTy = IV->getType();
6841 FunctionCallee DynamicInit = getKmpcForDynamicInitForType(Ty: IVTy, M, OMPBuilder&: *this);
6842 FunctionCallee DynamicNext = getKmpcForDynamicNextForType(Ty: IVTy, M, OMPBuilder&: *this);
6843
6844 // Allocate space for computed loop bounds as expected by the "init" function.
6845 Builder.SetInsertPoint(AllocaIP.getBlock()->getFirstNonPHIOrDbgOrAlloca());
6846 Type *I32Type = Type::getInt32Ty(C&: M.getContext());
6847 Value *PLastIter = Builder.CreateAlloca(Ty: I32Type, ArraySize: nullptr, Name: "p.lastiter");
6848 Value *PLowerBound = Builder.CreateAlloca(Ty: IVTy, ArraySize: nullptr, Name: "p.lowerbound");
6849 Value *PUpperBound = Builder.CreateAlloca(Ty: IVTy, ArraySize: nullptr, Name: "p.upperbound");
6850 Value *PStride = Builder.CreateAlloca(Ty: IVTy, ArraySize: nullptr, Name: "p.stride");
6851 CLI->setLastIter(PLastIter);
6852
6853 // At the end of the preheader, prepare for calling the "init" function by
6854 // storing the current loop bounds into the allocated space. A canonical loop
6855 // always iterates from 0 to trip-count with step 1. Note that "init" expects
6856 // and produces an inclusive upper bound.
6857 BasicBlock *PreHeader = CLI->getPreheader();
6858 Builder.SetInsertPoint(PreHeader->getTerminator());
6859 Constant *One = ConstantInt::get(Ty: IVTy, V: 1);
6860 Builder.CreateStore(Val: One, Ptr: PLowerBound);
6861 Value *UpperBound = CLI->getTripCount();
6862 Builder.CreateStore(Val: UpperBound, Ptr: PUpperBound);
6863 Builder.CreateStore(Val: One, Ptr: PStride);
6864
6865 BasicBlock *Header = CLI->getHeader();
6866 BasicBlock *Exit = CLI->getExit();
6867 BasicBlock *Cond = CLI->getCond();
6868 BasicBlock *Latch = CLI->getLatch();
6869 InsertPointTy AfterIP = CLI->getAfterIP();
6870
6871 // The CLI will be "broken" in the code below, as the loop is no longer
6872 // a valid canonical loop.
6873
6874 if (!Chunk)
6875 Chunk = One;
6876
6877 Value *ThreadNum =
6878 getOrCreateThreadID(Ident: getOrCreateIdent(SrcLocStr, SrcLocStrSize));
6879
6880 Constant *SchedulingType =
6881 ConstantInt::get(Ty: I32Type, V: static_cast<int>(SchedType));
6882
6883 // Call the "init" function.
6884 createRuntimeFunctionCall(Callee: DynamicInit, Args: {SrcLoc, ThreadNum, SchedulingType,
6885 /* LowerBound */ One, UpperBound,
6886 /* step */ One, Chunk});
6887
6888 // An outer loop around the existing one.
6889 BasicBlock *OuterCond = BasicBlock::Create(
6890 Context&: PreHeader->getContext(), Name: Twine(PreHeader->getName()) + ".outer.cond",
6891 Parent: PreHeader->getParent());
6892 // This needs to be 32-bit always, so can't use the IVTy Zero above.
6893 Builder.SetInsertPoint(TheBB: OuterCond, IP: OuterCond->getFirstInsertionPt());
6894 Value *Res = createRuntimeFunctionCall(
6895 Callee: DynamicNext,
6896 Args: {SrcLoc, ThreadNum, PLastIter, PLowerBound, PUpperBound, PStride});
6897 Constant *Zero32 = ConstantInt::get(Ty: I32Type, V: 0);
6898 Value *MoreWork = Builder.CreateCmp(Pred: CmpInst::ICMP_NE, LHS: Res, RHS: Zero32);
6899 Value *LowerBound =
6900 Builder.CreateSub(LHS: Builder.CreateLoad(Ty: IVTy, Ptr: PLowerBound), RHS: One, Name: "lb");
6901 Builder.CreateCondBr(Cond: MoreWork, True: Header, False: Exit);
6902
6903 // Change PHI-node in loop header to use outer cond rather than preheader,
6904 // and set IV to the LowerBound.
6905 Instruction *Phi = &Header->front();
6906 auto *PI = cast<PHINode>(Val: Phi);
6907 PI->setIncomingBlock(i: 0, BB: OuterCond);
6908 PI->setIncomingValue(i: 0, V: LowerBound);
6909
6910 // Then set the pre-header to jump to the OuterCond
6911 Instruction *Term = PreHeader->getTerminator();
6912 auto *Br = cast<UncondBrInst>(Val: Term);
6913 Br->setSuccessor(OuterCond);
6914
6915 // Modify the inner condition:
6916 // * Use the UpperBound returned from the DynamicNext call.
6917 // * jump to the loop outer loop when done with one of the inner loops.
6918 Builder.SetInsertPoint(TheBB: Cond, IP: Cond->getFirstInsertionPt());
6919 UpperBound = Builder.CreateLoad(Ty: IVTy, Ptr: PUpperBound, Name: "ub");
6920 Instruction *Comp = &*Builder.GetInsertPoint();
6921 auto *CI = cast<CmpInst>(Val: Comp);
6922 CI->setOperand(i_nocapture: 1, Val_nocapture: UpperBound);
6923 // Redirect the inner exit to branch to outer condition.
6924 Instruction *Branch = &Cond->back();
6925 auto *BI = cast<CondBrInst>(Val: Branch);
6926 assert(BI->getSuccessor(1) == Exit);
6927 BI->setSuccessor(idx: 1, NewSucc: OuterCond);
6928
6929 // Call the "fini" function if "ordered" is present in wsloop directive.
6930 if (Ordered) {
6931 Builder.SetInsertPoint(&Latch->back());
6932 FunctionCallee DynamicFini = getKmpcForDynamicFiniForType(Ty: IVTy, M, OMPBuilder&: *this);
6933 createRuntimeFunctionCall(Callee: DynamicFini, Args: {SrcLoc, ThreadNum});
6934 }
6935
6936 // Add the barrier if requested.
6937 if (NeedsBarrier) {
6938 Builder.SetInsertPoint(&Exit->back());
6939 InsertPointOrErrorTy BarrierIP =
6940 createBarrier(Loc: LocationDescription(Builder.saveIP(), DL),
6941 Kind: omp::Directive::OMPD_for, /* ForceSimpleCall */ false,
6942 /* CheckCancelFlag */ false);
6943 if (!BarrierIP)
6944 return BarrierIP.takeError();
6945 }
6946
6947 CLI->invalidate();
6948 return AfterIP;
6949}
6950
6951/// Redirect all edges that branch to \p OldTarget to \p NewTarget. That is,
6952/// after this \p OldTarget will be orphaned.
6953static void redirectAllPredecessorsTo(BasicBlock *OldTarget,
6954 BasicBlock *NewTarget, DebugLoc DL) {
6955 for (BasicBlock *Pred : make_early_inc_range(Range: predecessors(BB: OldTarget)))
6956 redirectTo(Source: Pred, Target: NewTarget, DL);
6957}
6958
6959static void removeUnusedBlocksFromParent(ArrayRef<BasicBlock *> BBs) {
6960 SmallPtrSet<BasicBlock *, 8> InternalBBs(from_range, BBs);
6961 // We add a block to BBsToKeep iff we have proven it has an external use.
6962 SmallPtrSet<BasicBlock *, 8> BBsToKeep;
6963
6964 while (true) {
6965 bool Changed = false;
6966
6967 for (BasicBlock *BB : BBs) {
6968 if (BBsToKeep.contains(Ptr: BB))
6969 continue;
6970
6971 for (Use &U : BB->uses()) {
6972 auto *UseInst = dyn_cast<Instruction>(Val: U.getUser());
6973 if (!UseInst)
6974 continue;
6975 BasicBlock *UseBB = UseInst->getParent();
6976 if (!InternalBBs.contains(Ptr: UseBB) || BBsToKeep.contains(Ptr: UseBB)) {
6977 BBsToKeep.insert(Ptr: BB);
6978 Changed = true;
6979 break;
6980 }
6981 }
6982 }
6983
6984 if (!Changed)
6985 break;
6986 }
6987
6988 SmallVector<BasicBlock *> BBsToDelete = filter_to_vector(
6989 C&: BBs, Pred: [&BBsToKeep](BasicBlock *BB) { return !BBsToKeep.contains(Ptr: BB); });
6990 DeleteDeadBlocks(BBs: BBsToDelete);
6991}
6992
6993CanonicalLoopInfo *
6994OpenMPIRBuilder::collapseLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
6995 InsertPointTy ComputeIP) {
6996 assert(Loops.size() >= 1 && "At least one loop required");
6997 size_t NumLoops = Loops.size();
6998
6999 // Nothing to do if there is already just one loop.
7000 if (NumLoops == 1)
7001 return Loops.front();
7002
7003 CanonicalLoopInfo *Outermost = Loops.front();
7004 CanonicalLoopInfo *Innermost = Loops.back();
7005 BasicBlock *OrigPreheader = Outermost->getPreheader();
7006 BasicBlock *OrigAfter = Outermost->getAfter();
7007 Function *F = OrigPreheader->getParent();
7008
7009 // Loop control blocks that may become orphaned later.
7010 SmallVector<BasicBlock *, 12> OldControlBBs;
7011 OldControlBBs.reserve(N: 6 * Loops.size());
7012 for (CanonicalLoopInfo *Loop : Loops)
7013 Loop->collectControlBlocks(BBs&: OldControlBBs);
7014
7015 // Setup the IRBuilder for inserting the trip count computation.
7016 Builder.SetCurrentDebugLocation(DL);
7017 if (ComputeIP.isSet())
7018 Builder.restoreIP(IP: ComputeIP);
7019 else
7020 Builder.restoreIP(IP: Outermost->getPreheaderIP());
7021
7022 // Derive the collapsed' loop trip count.
7023 // TODO: Find common/largest indvar type.
7024 Value *CollapsedTripCount = nullptr;
7025 for (CanonicalLoopInfo *L : Loops) {
7026 assert(L->isValid() &&
7027 "All loops to collapse must be valid canonical loops");
7028 Value *OrigTripCount = L->getTripCount();
7029 if (!CollapsedTripCount) {
7030 CollapsedTripCount = OrigTripCount;
7031 continue;
7032 }
7033
7034 // TODO: Enable UndefinedSanitizer to diagnose an overflow here.
7035 CollapsedTripCount =
7036 Builder.CreateNUWMul(LHS: CollapsedTripCount, RHS: OrigTripCount);
7037 }
7038
7039 // Create the collapsed loop control flow.
7040 CanonicalLoopInfo *Result =
7041 createLoopSkeleton(DL, TripCount: CollapsedTripCount, F,
7042 PreInsertBefore: OrigPreheader->getNextNode(), PostInsertBefore: OrigAfter, Name: "collapsed",
7043 /*IsCollapsed=*/true);
7044
7045 // Build the collapsed loop body code.
7046 // Start with deriving the input loop induction variables from the collapsed
7047 // one, using a divmod scheme. To preserve the original loops' order, the
7048 // innermost loop use the least significant bits.
7049 Builder.restoreIP(IP: Result->getBodyIP());
7050
7051 Value *Leftover = Result->getIndVar();
7052 SmallVector<Value *> NewIndVars;
7053 NewIndVars.resize(N: NumLoops);
7054 for (int i = NumLoops - 1; i >= 1; --i) {
7055 Value *OrigTripCount = Loops[i]->getTripCount();
7056
7057 Value *NewIndVar = Builder.CreateURem(LHS: Leftover, RHS: OrigTripCount);
7058 NewIndVars[i] = NewIndVar;
7059
7060 Leftover = Builder.CreateUDiv(LHS: Leftover, RHS: OrigTripCount);
7061 }
7062 // Outermost loop gets all the remaining bits.
7063 NewIndVars[0] = Leftover;
7064
7065 // Construct the loop body control flow.
7066 // We progressively construct the branch structure following in direction of
7067 // the control flow, from the leading in-between code, the loop nest body, the
7068 // trailing in-between code, and rejoining the collapsed loop's latch.
7069 // ContinueBlock and ContinuePred keep track of the source(s) of next edge. If
7070 // the ContinueBlock is set, continue with that block. If ContinuePred, use
7071 // its predecessors as sources.
7072 BasicBlock *ContinueBlock = Result->getBody();
7073 BasicBlock *ContinuePred = nullptr;
7074 auto ContinueWith = [&ContinueBlock, &ContinuePred, DL](BasicBlock *Dest,
7075 BasicBlock *NextSrc) {
7076 if (ContinueBlock)
7077 redirectTo(Source: ContinueBlock, Target: Dest, DL);
7078 else
7079 redirectAllPredecessorsTo(OldTarget: ContinuePred, NewTarget: Dest, DL);
7080
7081 ContinueBlock = nullptr;
7082 ContinuePred = NextSrc;
7083 };
7084
7085 // The code before the nested loop of each level.
7086 // Because we are sinking it into the nest, it will be executed more often
7087 // that the original loop. More sophisticated schemes could keep track of what
7088 // the in-between code is and instantiate it only once per thread.
7089 for (size_t i = 0; i < NumLoops - 1; ++i)
7090 ContinueWith(Loops[i]->getBody(), Loops[i + 1]->getHeader());
7091
7092 // Connect the loop nest body.
7093 ContinueWith(Innermost->getBody(), Innermost->getLatch());
7094
7095 // The code after the nested loop at each level.
7096 for (size_t i = NumLoops - 1; i > 0; --i)
7097 ContinueWith(Loops[i]->getAfter(), Loops[i - 1]->getLatch());
7098
7099 // Connect the finished loop to the collapsed loop latch.
7100 ContinueWith(Result->getLatch(), nullptr);
7101
7102 // Replace the input loops with the new collapsed loop.
7103 redirectTo(Source: Outermost->getPreheader(), Target: Result->getPreheader(), DL);
7104 redirectTo(Source: Result->getAfter(), Target: Outermost->getAfter(), DL);
7105
7106 // Replace the input loop indvars with the derived ones.
7107 for (size_t i = 0; i < NumLoops; ++i)
7108 Loops[i]->getIndVar()->replaceAllUsesWith(V: NewIndVars[i]);
7109
7110 // Remove unused parts of the input loops.
7111 removeUnusedBlocksFromParent(BBs: OldControlBBs);
7112
7113 for (CanonicalLoopInfo *L : Loops)
7114 L->invalidate();
7115
7116#ifndef NDEBUG
7117 Result->assertOK();
7118#endif
7119 return Result;
7120}
7121
7122std::vector<CanonicalLoopInfo *>
7123OpenMPIRBuilder::tileLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
7124 ArrayRef<Value *> TileSizes) {
7125 assert(TileSizes.size() == Loops.size() &&
7126 "Must pass as many tile sizes as there are loops");
7127 int NumLoops = Loops.size();
7128 assert(NumLoops >= 1 && "At least one loop to tile required");
7129
7130 CanonicalLoopInfo *OutermostLoop = Loops.front();
7131 CanonicalLoopInfo *InnermostLoop = Loops.back();
7132 Function *F = OutermostLoop->getBody()->getParent();
7133 BasicBlock *InnerEnter = InnermostLoop->getBody();
7134 BasicBlock *InnerLatch = InnermostLoop->getLatch();
7135
7136 // Loop control blocks that may become orphaned later.
7137 SmallVector<BasicBlock *, 12> OldControlBBs;
7138 OldControlBBs.reserve(N: 6 * Loops.size());
7139 for (CanonicalLoopInfo *Loop : Loops)
7140 Loop->collectControlBlocks(BBs&: OldControlBBs);
7141
7142 // Collect original trip counts and induction variable to be accessible by
7143 // index. Also, the structure of the original loops is not preserved during
7144 // the construction of the tiled loops, so do it before we scavenge the BBs of
7145 // any original CanonicalLoopInfo.
7146 SmallVector<Value *, 4> OrigTripCounts, OrigIndVars;
7147 for (CanonicalLoopInfo *L : Loops) {
7148 assert(L->isValid() && "All input loops must be valid canonical loops");
7149 OrigTripCounts.push_back(Elt: L->getTripCount());
7150 OrigIndVars.push_back(Elt: L->getIndVar());
7151 }
7152
7153 // Collect the code between loop headers. These may contain SSA definitions
7154 // that are used in the loop nest body. To be usable with in the innermost
7155 // body, these BasicBlocks will be sunk into the loop nest body. That is,
7156 // these instructions may be executed more often than before the tiling.
7157 // TODO: It would be sufficient to only sink them into body of the
7158 // corresponding tile loop.
7159 SmallVector<std::pair<BasicBlock *, BasicBlock *>, 4> InbetweenCode;
7160 for (int i = 0; i < NumLoops - 1; ++i) {
7161 CanonicalLoopInfo *Surrounding = Loops[i];
7162 CanonicalLoopInfo *Nested = Loops[i + 1];
7163
7164 BasicBlock *EnterBB = Surrounding->getBody();
7165 BasicBlock *ExitBB = Nested->getHeader();
7166 InbetweenCode.emplace_back(Args&: EnterBB, Args&: ExitBB);
7167 }
7168
7169 // Compute the trip counts of the floor loops.
7170 Builder.SetCurrentDebugLocation(DL);
7171 Builder.restoreIP(IP: OutermostLoop->getPreheaderIP());
7172 SmallVector<Value *, 4> FloorCompleteCount, FloorCount, FloorRems;
7173 for (int i = 0; i < NumLoops; ++i) {
7174 Value *TileSize = TileSizes[i];
7175 Value *OrigTripCount = OrigTripCounts[i];
7176 Type *IVType = OrigTripCount->getType();
7177
7178 Value *FloorCompleteTripCount = Builder.CreateUDiv(LHS: OrigTripCount, RHS: TileSize);
7179 Value *FloorTripRem = Builder.CreateURem(LHS: OrigTripCount, RHS: TileSize);
7180
7181 // 0 if tripcount divides the tilesize, 1 otherwise.
7182 // 1 means we need an additional iteration for a partial tile.
7183 //
7184 // Unfortunately we cannot just use the roundup-formula
7185 // (tripcount + tilesize - 1)/tilesize
7186 // because the summation might overflow. We do not want introduce undefined
7187 // behavior when the untiled loop nest did not.
7188 Value *FloorTripOverflow =
7189 Builder.CreateICmpNE(LHS: FloorTripRem, RHS: ConstantInt::get(Ty: IVType, V: 0));
7190
7191 FloorTripOverflow = Builder.CreateZExt(V: FloorTripOverflow, DestTy: IVType);
7192 Value *FloorTripCount =
7193 Builder.CreateAdd(LHS: FloorCompleteTripCount, RHS: FloorTripOverflow,
7194 Name: "omp_floor" + Twine(i) + ".tripcount", HasNUW: true);
7195
7196 // Remember some values for later use.
7197 FloorCompleteCount.push_back(Elt: FloorCompleteTripCount);
7198 FloorCount.push_back(Elt: FloorTripCount);
7199 FloorRems.push_back(Elt: FloorTripRem);
7200 }
7201
7202 // Generate the new loop nest, from the outermost to the innermost.
7203 std::vector<CanonicalLoopInfo *> Result;
7204 Result.reserve(n: NumLoops * 2);
7205
7206 // The basic block of the surrounding loop that enters the nest generated
7207 // loop.
7208 BasicBlock *Enter = OutermostLoop->getPreheader();
7209
7210 // The basic block of the surrounding loop where the inner code should
7211 // continue.
7212 BasicBlock *Continue = OutermostLoop->getAfter();
7213
7214 // Where the next loop basic block should be inserted.
7215 BasicBlock *OutroInsertBefore = InnermostLoop->getExit();
7216
7217 auto EmbeddNewLoop =
7218 [this, DL, F, InnerEnter, &Enter, &Continue, &OutroInsertBefore](
7219 Value *TripCount, const Twine &Name) -> CanonicalLoopInfo * {
7220 CanonicalLoopInfo *EmbeddedLoop = createLoopSkeleton(
7221 DL, TripCount, F, PreInsertBefore: InnerEnter, PostInsertBefore: OutroInsertBefore, Name);
7222 redirectTo(Source: Enter, Target: EmbeddedLoop->getPreheader(), DL);
7223 redirectTo(Source: EmbeddedLoop->getAfter(), Target: Continue, DL);
7224
7225 // Setup the position where the next embedded loop connects to this loop.
7226 Enter = EmbeddedLoop->getBody();
7227 Continue = EmbeddedLoop->getLatch();
7228 OutroInsertBefore = EmbeddedLoop->getLatch();
7229 return EmbeddedLoop;
7230 };
7231
7232 auto EmbeddNewLoops = [&Result, &EmbeddNewLoop](ArrayRef<Value *> TripCounts,
7233 const Twine &NameBase) {
7234 for (auto P : enumerate(First&: TripCounts)) {
7235 CanonicalLoopInfo *EmbeddedLoop =
7236 EmbeddNewLoop(P.value(), NameBase + Twine(P.index()));
7237 Result.push_back(x: EmbeddedLoop);
7238 }
7239 };
7240
7241 EmbeddNewLoops(FloorCount, "floor");
7242
7243 // Within the innermost floor loop, emit the code that computes the tile
7244 // sizes.
7245 Builder.SetInsertPoint(Enter->getTerminator());
7246 SmallVector<Value *, 4> TileCounts;
7247 for (int i = 0; i < NumLoops; ++i) {
7248 CanonicalLoopInfo *FloorLoop = Result[i];
7249 Value *TileSize = TileSizes[i];
7250
7251 Value *FloorIsEpilogue =
7252 Builder.CreateICmpEQ(LHS: FloorLoop->getIndVar(), RHS: FloorCompleteCount[i]);
7253 Value *TileTripCount =
7254 Builder.CreateSelect(C: FloorIsEpilogue, True: FloorRems[i], False: TileSize);
7255
7256 TileCounts.push_back(Elt: TileTripCount);
7257 }
7258
7259 // Create the tile loops.
7260 EmbeddNewLoops(TileCounts, "tile");
7261
7262 // Insert the inbetween code into the body.
7263 BasicBlock *BodyEnter = Enter;
7264 BasicBlock *BodyEntered = nullptr;
7265 for (std::pair<BasicBlock *, BasicBlock *> P : InbetweenCode) {
7266 BasicBlock *EnterBB = P.first;
7267 BasicBlock *ExitBB = P.second;
7268
7269 if (BodyEnter)
7270 redirectTo(Source: BodyEnter, Target: EnterBB, DL);
7271 else
7272 redirectAllPredecessorsTo(OldTarget: BodyEntered, NewTarget: EnterBB, DL);
7273
7274 BodyEnter = nullptr;
7275 BodyEntered = ExitBB;
7276 }
7277
7278 // Append the original loop nest body into the generated loop nest body.
7279 if (BodyEnter)
7280 redirectTo(Source: BodyEnter, Target: InnerEnter, DL);
7281 else
7282 redirectAllPredecessorsTo(OldTarget: BodyEntered, NewTarget: InnerEnter, DL);
7283 redirectAllPredecessorsTo(OldTarget: InnerLatch, NewTarget: Continue, DL);
7284
7285 // Replace the original induction variable with an induction variable computed
7286 // from the tile and floor induction variables.
7287 Builder.restoreIP(IP: Result.back()->getBodyIP());
7288 for (int i = 0; i < NumLoops; ++i) {
7289 CanonicalLoopInfo *FloorLoop = Result[i];
7290 CanonicalLoopInfo *TileLoop = Result[NumLoops + i];
7291 Value *OrigIndVar = OrigIndVars[i];
7292 Value *Size = TileSizes[i];
7293
7294 Value *Scale =
7295 Builder.CreateMul(LHS: Size, RHS: FloorLoop->getIndVar(), Name: {}, /*HasNUW=*/true);
7296 Value *Shift =
7297 Builder.CreateAdd(LHS: Scale, RHS: TileLoop->getIndVar(), Name: {}, /*HasNUW=*/true);
7298 OrigIndVar->replaceAllUsesWith(V: Shift);
7299 }
7300
7301 // Remove unused parts of the original loops.
7302 removeUnusedBlocksFromParent(BBs: OldControlBBs);
7303
7304 for (CanonicalLoopInfo *L : Loops)
7305 L->invalidate();
7306
7307#ifndef NDEBUG
7308 for (CanonicalLoopInfo *GenL : Result)
7309 GenL->assertOK();
7310#endif
7311 return Result;
7312}
7313
7314/// Attach metadata \p Properties to the basic block described by \p BB. If the
7315/// basic block already has metadata, the basic block properties are appended.
7316static void addBasicBlockMetadata(BasicBlock *BB,
7317 ArrayRef<Metadata *> Properties) {
7318 // Nothing to do if no property to attach.
7319 if (Properties.empty())
7320 return;
7321
7322 LLVMContext &Ctx = BB->getContext();
7323 SmallVector<Metadata *> NewProperties;
7324 NewProperties.push_back(Elt: nullptr);
7325
7326 // If the basic block already has metadata, prepend it to the new metadata.
7327 MDNode *Existing = BB->getTerminator()->getMetadata(KindID: LLVMContext::MD_loop);
7328 if (Existing)
7329 append_range(C&: NewProperties, R: drop_begin(RangeOrContainer: Existing->operands(), N: 1));
7330
7331 append_range(C&: NewProperties, R&: Properties);
7332 MDNode *BasicBlockID = MDNode::getDistinct(Context&: Ctx, MDs: NewProperties);
7333 BasicBlockID->replaceOperandWith(I: 0, New: BasicBlockID);
7334
7335 BB->getTerminator()->setMetadata(KindID: LLVMContext::MD_loop, Node: BasicBlockID);
7336}
7337
7338/// Attach loop metadata \p Properties to the loop described by \p Loop. If the
7339/// loop already has metadata, the loop properties are appended.
7340static void addLoopMetadata(CanonicalLoopInfo *Loop,
7341 ArrayRef<Metadata *> Properties) {
7342 assert(Loop->isValid() && "Expecting a valid CanonicalLoopInfo");
7343
7344 // Attach metadata to the loop's latch
7345 BasicBlock *Latch = Loop->getLatch();
7346 assert(Latch && "A valid CanonicalLoopInfo must have a unique latch");
7347 addBasicBlockMetadata(BB: Latch, Properties);
7348}
7349
7350/// Attach llvm.access.group metadata to the memref instructions of \p Block
7351static void addAccessGroupMetadata(BasicBlock *Block, MDNode *AccessGroup,
7352 LoopInfo &LI) {
7353 for (Instruction &I : *Block) {
7354 if (I.mayReadOrWriteMemory()) {
7355 // TODO: This instruction may already have access group from
7356 // other pragmas e.g. #pragma clang loop vectorize. Append
7357 // so that the existing metadata is not overwritten.
7358 I.setMetadata(KindID: LLVMContext::MD_access_group, Node: AccessGroup);
7359 }
7360 }
7361}
7362
7363CanonicalLoopInfo *
7364OpenMPIRBuilder::fuseLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops) {
7365 CanonicalLoopInfo *firstLoop = Loops.front();
7366 CanonicalLoopInfo *lastLoop = Loops.back();
7367 Function *F = firstLoop->getPreheader()->getParent();
7368
7369 // Loop control blocks that will become orphaned later
7370 SmallVector<BasicBlock *> oldControlBBs;
7371 for (CanonicalLoopInfo *Loop : Loops)
7372 Loop->collectControlBlocks(BBs&: oldControlBBs);
7373
7374 // Collect original trip counts
7375 SmallVector<Value *> origTripCounts;
7376 for (CanonicalLoopInfo *L : Loops) {
7377 assert(L->isValid() && "All input loops must be valid canonical loops");
7378 origTripCounts.push_back(Elt: L->getTripCount());
7379 }
7380
7381 Builder.SetCurrentDebugLocation(DL);
7382
7383 // Compute max trip count.
7384 // The fused loop will be from 0 to max(origTripCounts)
7385 BasicBlock *TCBlock = BasicBlock::Create(Context&: F->getContext(), Name: "omp.fuse.comp.tc",
7386 Parent: F, InsertBefore: firstLoop->getHeader());
7387 Builder.SetInsertPoint(TCBlock);
7388 Value *fusedTripCount = nullptr;
7389 for (CanonicalLoopInfo *L : Loops) {
7390 assert(L->isValid() && "All loops to fuse must be valid canonical loops");
7391 Value *origTripCount = L->getTripCount();
7392 if (!fusedTripCount) {
7393 fusedTripCount = origTripCount;
7394 continue;
7395 }
7396 Value *condTP = Builder.CreateICmpSGT(LHS: fusedTripCount, RHS: origTripCount);
7397 fusedTripCount = Builder.CreateSelect(C: condTP, True: fusedTripCount, False: origTripCount,
7398 Name: ".omp.fuse.tc");
7399 }
7400
7401 // Generate new loop
7402 CanonicalLoopInfo *fused =
7403 createLoopSkeleton(DL, TripCount: fusedTripCount, F, PreInsertBefore: firstLoop->getBody(),
7404 PostInsertBefore: lastLoop->getLatch(), Name: "fused");
7405
7406 // Replace original loops with the fused loop
7407 // Preheader and After are not considered inside the CLI.
7408 // These are used to compute the individual TCs of the loops
7409 // so they have to be put before the resulting fused loop.
7410 // Moving them up for readability.
7411 for (size_t i = 0; i < Loops.size() - 1; ++i) {
7412 Loops[i]->getPreheader()->moveBefore(MovePos: TCBlock);
7413 Loops[i]->getAfter()->moveBefore(MovePos: TCBlock);
7414 }
7415 lastLoop->getPreheader()->moveBefore(MovePos: TCBlock);
7416
7417 for (size_t i = 0; i < Loops.size() - 1; ++i) {
7418 redirectTo(Source: Loops[i]->getPreheader(), Target: Loops[i]->getAfter(), DL);
7419 redirectTo(Source: Loops[i]->getAfter(), Target: Loops[i + 1]->getPreheader(), DL);
7420 }
7421 redirectTo(Source: lastLoop->getPreheader(), Target: TCBlock, DL);
7422 redirectTo(Source: TCBlock, Target: fused->getPreheader(), DL);
7423 redirectTo(Source: fused->getAfter(), Target: lastLoop->getAfter(), DL);
7424
7425 // Build the fused body
7426 // Create new Blocks with conditions that jump to the original loop bodies
7427 SmallVector<BasicBlock *> condBBs;
7428 SmallVector<Value *> condValues;
7429 for (size_t i = 0; i < Loops.size(); ++i) {
7430 BasicBlock *condBlock = BasicBlock::Create(
7431 Context&: F->getContext(), Name: "omp.fused.inner.cond", Parent: F, InsertBefore: Loops[i]->getBody());
7432 Builder.SetInsertPoint(condBlock);
7433 Value *condValue =
7434 Builder.CreateICmpSLT(LHS: fused->getIndVar(), RHS: origTripCounts[i]);
7435 condBBs.push_back(Elt: condBlock);
7436 condValues.push_back(Elt: condValue);
7437 }
7438 // Join the condition blocks with the bodies of the original loops
7439 redirectTo(Source: fused->getBody(), Target: condBBs[0], DL);
7440 for (size_t i = 0; i < Loops.size() - 1; ++i) {
7441 Builder.SetInsertPoint(condBBs[i]);
7442 Builder.CreateCondBr(Cond: condValues[i], True: Loops[i]->getBody(), False: condBBs[i + 1]);
7443 redirectAllPredecessorsTo(OldTarget: Loops[i]->getLatch(), NewTarget: condBBs[i + 1], DL);
7444 // Replace the IV with the fused IV
7445 Loops[i]->getIndVar()->replaceAllUsesWith(V: fused->getIndVar());
7446 }
7447 // Last body jumps to the created end body block
7448 Builder.SetInsertPoint(condBBs.back());
7449 Builder.CreateCondBr(Cond: condValues.back(), True: lastLoop->getBody(),
7450 False: fused->getLatch());
7451 redirectAllPredecessorsTo(OldTarget: lastLoop->getLatch(), NewTarget: fused->getLatch(), DL);
7452 // Replace the IV with the fused IV
7453 lastLoop->getIndVar()->replaceAllUsesWith(V: fused->getIndVar());
7454
7455 // The loop latch must have only one predecessor. Currently it is branched to
7456 // from both the last condition block and the last loop body
7457 fused->getLatch()->splitBasicBlockBefore(I: fused->getLatch()->begin(),
7458 BBName: "omp.fused.pre_latch");
7459
7460 // Remove unused parts
7461 removeUnusedBlocksFromParent(BBs: oldControlBBs);
7462
7463 // Invalidate old CLIs
7464 for (CanonicalLoopInfo *L : Loops)
7465 L->invalidate();
7466
7467#ifndef NDEBUG
7468 fused->assertOK();
7469#endif
7470 return fused;
7471}
7472
7473void OpenMPIRBuilder::unrollLoopFull(DebugLoc, CanonicalLoopInfo *Loop) {
7474 LLVMContext &Ctx = Builder.getContext();
7475 addLoopMetadata(
7476 Loop, Properties: {MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.enable")),
7477 MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.full"))});
7478}
7479
7480void OpenMPIRBuilder::unrollLoopHeuristic(DebugLoc, CanonicalLoopInfo *Loop) {
7481 LLVMContext &Ctx = Builder.getContext();
7482 addLoopMetadata(
7483 Loop, Properties: {
7484 MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.enable")),
7485 });
7486}
7487
7488void OpenMPIRBuilder::createIfVersion(CanonicalLoopInfo *CanonicalLoop,
7489 Value *IfCond, ValueToValueMapTy &VMap,
7490 LoopAnalysis &LIA, LoopInfo &LI, Loop *L,
7491 const Twine &NamePrefix) {
7492 Function *F = CanonicalLoop->getFunction();
7493
7494 // We can't do
7495 // if (cond) {
7496 // simd_loop;
7497 // } else {
7498 // non_simd_loop;
7499 // }
7500 // because then the CanonicalLoopInfo would only point to one of the loops:
7501 // leading to other constructs operating on the same loop to malfunction.
7502 // Instead generate
7503 // while (...) {
7504 // if (cond) {
7505 // simd_body;
7506 // } else {
7507 // not_simd_body;
7508 // }
7509 // }
7510 // At least for simple loops, LLVM seems able to hoist the if out of the loop
7511 // body at -O3
7512
7513 // Define where if branch should be inserted
7514 auto SplitBeforeIt = CanonicalLoop->getBody()->getFirstNonPHIIt();
7515
7516 // Create additional blocks for the if statement
7517 BasicBlock *Cond = SplitBeforeIt->getParent();
7518 llvm::LLVMContext &C = Cond->getContext();
7519 llvm::BasicBlock *ThenBlock = llvm::BasicBlock::Create(
7520 Context&: C, Name: NamePrefix + ".if.then", Parent: Cond->getParent(), InsertBefore: Cond->getNextNode());
7521 llvm::BasicBlock *ElseBlock = llvm::BasicBlock::Create(
7522 Context&: C, Name: NamePrefix + ".if.else", Parent: Cond->getParent(), InsertBefore: CanonicalLoop->getExit());
7523
7524 // Create if condition branch.
7525 Builder.SetInsertPoint(SplitBeforeIt);
7526 Instruction *BrInstr =
7527 Builder.CreateCondBr(Cond: IfCond, True: ThenBlock, /*ifFalse*/ False: ElseBlock);
7528 InsertPointTy IP{BrInstr->getParent(), ++BrInstr->getIterator()};
7529 // Then block contains branch to omp loop body which needs to be vectorized
7530 spliceBB(IP, New: ThenBlock, CreateBranch: false, DL: Builder.getCurrentDebugLocation());
7531 ThenBlock->replaceSuccessorsPhiUsesWith(Old: Cond, New: ThenBlock);
7532
7533 Builder.SetInsertPoint(ElseBlock);
7534
7535 // Clone loop for the else branch
7536 SmallVector<BasicBlock *, 8> NewBlocks;
7537
7538 SmallVector<BasicBlock *, 8> ExistingBlocks;
7539 ExistingBlocks.reserve(N: L->getNumBlocks() + 1);
7540 ExistingBlocks.push_back(Elt: ThenBlock);
7541 ExistingBlocks.append(in_start: L->block_begin(), in_end: L->block_end());
7542 // Cond is the block that has the if clause condition
7543 // LoopCond is omp_loop.cond
7544 // LoopHeader is omp_loop.header
7545 BasicBlock *LoopCond = Cond->getUniquePredecessor();
7546 BasicBlock *LoopHeader = LoopCond->getUniquePredecessor();
7547 assert(LoopCond && LoopHeader && "Invalid loop structure");
7548 for (BasicBlock *Block : ExistingBlocks) {
7549 if (Block == L->getLoopPreheader() || Block == L->getLoopLatch() ||
7550 Block == LoopHeader || Block == LoopCond || Block == Cond) {
7551 continue;
7552 }
7553 BasicBlock *NewBB = CloneBasicBlock(BB: Block, VMap, NameSuffix: "", F);
7554
7555 // fix name not to be omp.if.then
7556 if (Block == ThenBlock)
7557 NewBB->setName(NamePrefix + ".if.else");
7558
7559 NewBB->moveBefore(MovePos: CanonicalLoop->getExit());
7560 VMap[Block] = NewBB;
7561 NewBlocks.push_back(Elt: NewBB);
7562 }
7563 remapInstructionsInBlocks(Blocks: NewBlocks, VMap);
7564 Builder.CreateBr(Dest: NewBlocks.front());
7565
7566 // The loop latch must have only one predecessor. Currently it is branched to
7567 // from both the 'then' and 'else' branches.
7568 L->getLoopLatch()->splitBasicBlockBefore(I: L->getLoopLatch()->begin(),
7569 BBName: NamePrefix + ".pre_latch");
7570
7571 // Ensure that the then block is added to the loop so we add the attributes in
7572 // the next step
7573 L->addBasicBlockToLoop(NewBB: ThenBlock, LI);
7574}
7575
7576unsigned
7577OpenMPIRBuilder::getOpenMPDefaultSimdAlign(const Triple &TargetTriple,
7578 const StringMap<bool> &Features) {
7579 if (TargetTriple.isX86()) {
7580 if (Features.lookup(Key: "avx512f"))
7581 return 512;
7582 else if (Features.lookup(Key: "avx"))
7583 return 256;
7584 return 128;
7585 }
7586 if (TargetTriple.isPPC())
7587 return 128;
7588 if (TargetTriple.isWasm())
7589 return 128;
7590 if (TargetTriple.isSystemZ())
7591 return 64;
7592 return 0;
7593}
7594
7595void OpenMPIRBuilder::applySimd(CanonicalLoopInfo *CanonicalLoop,
7596 MapVector<Value *, Value *> AlignedVars,
7597 Value *IfCond, OrderKind Order,
7598 ConstantInt *Simdlen, ConstantInt *Safelen) {
7599 LLVMContext &Ctx = Builder.getContext();
7600
7601 Function *F = CanonicalLoop->getFunction();
7602
7603 // Blocks must have terminators.
7604 // FIXME: Don't run analyses on incomplete/invalid IR.
7605 SmallVector<Instruction *> UIs;
7606 for (BasicBlock &BB : *F)
7607 if (!BB.hasTerminator())
7608 UIs.push_back(Elt: new UnreachableInst(F->getContext(), &BB));
7609
7610 // TODO: We should not rely on pass manager. Currently we use pass manager
7611 // only for getting llvm::Loop which corresponds to given CanonicalLoopInfo
7612 // object. We should have a method which returns all blocks between
7613 // CanonicalLoopInfo::getHeader() and CanonicalLoopInfo::getAfter()
7614 FunctionAnalysisManager FAM;
7615 FAM.registerPass(PassBuilder: []() { return DominatorTreeAnalysis(); });
7616 FAM.registerPass(PassBuilder: []() { return LoopAnalysis(); });
7617 FAM.registerPass(PassBuilder: []() { return PassInstrumentationAnalysis(); });
7618
7619 LoopAnalysis LIA;
7620 LoopInfo &&LI = LIA.run(F&: *F, AM&: FAM);
7621
7622 for (Instruction *I : UIs)
7623 I->eraseFromParent();
7624
7625 Loop *L = LI.getLoopFor(BB: CanonicalLoop->getHeader());
7626 if (AlignedVars.size()) {
7627 InsertPointTy IP = Builder.saveIP();
7628 for (auto &AlignedItem : AlignedVars) {
7629 Value *AlignedPtr = AlignedItem.first;
7630 Value *Alignment = AlignedItem.second;
7631 Instruction *loadInst = dyn_cast<Instruction>(Val: AlignedPtr);
7632 Builder.SetInsertPoint(loadInst->getNextNode());
7633 Builder.CreateAlignmentAssumption(DL: F->getDataLayout(), PtrValue: AlignedPtr,
7634 Alignment);
7635 }
7636 Builder.restoreIP(IP);
7637 }
7638
7639 if (IfCond) {
7640 ValueToValueMapTy VMap;
7641 createIfVersion(CanonicalLoop, IfCond, VMap, LIA, LI, L, NamePrefix: "simd");
7642 }
7643
7644 SmallPtrSet<BasicBlock *, 8> Reachable;
7645
7646 // Get the basic blocks from the loop in which memref instructions
7647 // can be found.
7648 // TODO: Generalize getting all blocks inside a CanonicalizeLoopInfo,
7649 // preferably without running any passes.
7650 for (BasicBlock *Block : L->getBlocks()) {
7651 if (Block == CanonicalLoop->getCond() ||
7652 Block == CanonicalLoop->getHeader())
7653 continue;
7654 Reachable.insert(Ptr: Block);
7655 }
7656
7657 SmallVector<Metadata *> LoopMDList;
7658
7659 // In presence of finite 'safelen', it may be unsafe to mark all
7660 // the memory instructions parallel, because loop-carried
7661 // dependences of 'safelen' iterations are possible.
7662 // If clause order(concurrent) is specified then the memory instructions
7663 // are marked parallel even if 'safelen' is finite.
7664 if ((Safelen == nullptr) || (Order == OrderKind::OMP_ORDER_concurrent))
7665 applyParallelAccessesMetadata(CLI: CanonicalLoop, Ctx, Loop: L, LoopInfo&: LI, LoopMDList);
7666
7667 // FIXME: the IF clause shares a loop backedge for the SIMD and non-SIMD
7668 // versions so we can't add the loop attributes in that case.
7669 if (IfCond) {
7670 // we can still add llvm.loop.parallel_access
7671 addLoopMetadata(Loop: CanonicalLoop, Properties: LoopMDList);
7672 return;
7673 }
7674
7675 // Use the above access group metadata to create loop level
7676 // metadata, which should be distinct for each loop.
7677 LoopMDList.push_back(
7678 Elt: MDNode::get(Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.vectorize.enable")}));
7679
7680 if (Simdlen || Safelen) {
7681 // If both simdlen and safelen clauses are specified, the value of the
7682 // simdlen parameter must be less than or equal to the value of the safelen
7683 // parameter. Therefore, use safelen only in the absence of simdlen.
7684 ConstantInt *VectorizeWidth = Simdlen == nullptr ? Safelen : Simdlen;
7685 LoopMDList.push_back(
7686 Elt: MDNode::get(Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.vectorize.width"),
7687 ConstantAsMetadata::get(C: VectorizeWidth)}));
7688 }
7689
7690 addLoopMetadata(Loop: CanonicalLoop, Properties: LoopMDList);
7691}
7692
7693/// Create the TargetMachine object to query the backend for optimization
7694/// preferences.
7695///
7696/// Ideally, this would be passed from the front-end to the OpenMPBuilder, but
7697/// e.g. Clang does not pass it to its CodeGen layer and creates it only when
7698/// needed for the LLVM pass pipline. We use some default options to avoid
7699/// having to pass too many settings from the frontend that probably do not
7700/// matter.
7701///
7702/// Currently, TargetMachine is only used sometimes by the unrollLoopPartial
7703/// method. If we are going to use TargetMachine for more purposes, especially
7704/// those that are sensitive to TargetOptions, RelocModel and CodeModel, it
7705/// might become be worth requiring front-ends to pass on their TargetMachine,
7706/// or at least cache it between methods. Note that while fontends such as Clang
7707/// have just a single main TargetMachine per translation unit, "target-cpu" and
7708/// "target-features" that determine the TargetMachine are per-function and can
7709/// be overrided using __attribute__((target("OPTIONS"))).
7710static std::unique_ptr<TargetMachine>
7711createTargetMachine(Function *F, CodeGenOptLevel OptLevel) {
7712 Module *M = F->getParent();
7713
7714 StringRef CPU = F->getFnAttribute(Kind: "target-cpu").getValueAsString();
7715 StringRef Features = F->getFnAttribute(Kind: "target-features").getValueAsString();
7716 const llvm::Triple &Triple = M->getTargetTriple();
7717
7718 std::string Error;
7719 const llvm::Target *TheTarget = TargetRegistry::lookupTarget(TheTriple: Triple, Error);
7720 if (!TheTarget)
7721 return {};
7722
7723 llvm::TargetOptions Options;
7724 return std::unique_ptr<TargetMachine>(TheTarget->createTargetMachine(
7725 TT: Triple, CPU, Features, Options, /*RelocModel=*/RM: std::nullopt,
7726 /*CodeModel=*/CM: std::nullopt, OL: OptLevel));
7727}
7728
7729/// Heuristically determine the best-performant unroll factor for \p CLI. This
7730/// depends on the target processor. We are re-using the same heuristics as the
7731/// LoopUnrollPass.
7732static int32_t computeHeuristicUnrollFactor(CanonicalLoopInfo *CLI) {
7733 Function *F = CLI->getFunction();
7734
7735 // Assume the user requests the most aggressive unrolling, even if the rest of
7736 // the code is optimized using a lower setting.
7737 CodeGenOptLevel OptLevel = CodeGenOptLevel::Aggressive;
7738 std::unique_ptr<TargetMachine> TM = createTargetMachine(F, OptLevel);
7739
7740 // Blocks must have terminators.
7741 // FIXME: Don't run analyses on incomplete/invalid IR.
7742 SmallVector<Instruction *> UIs;
7743 for (BasicBlock &BB : *F)
7744 if (!BB.hasTerminator())
7745 UIs.push_back(Elt: new UnreachableInst(F->getContext(), &BB));
7746
7747 FunctionAnalysisManager FAM;
7748 FAM.registerPass(PassBuilder: []() { return TargetLibraryAnalysis(); });
7749 FAM.registerPass(PassBuilder: []() { return AssumptionAnalysis(); });
7750 FAM.registerPass(PassBuilder: []() { return DominatorTreeAnalysis(); });
7751 FAM.registerPass(PassBuilder: []() { return LoopAnalysis(); });
7752 FAM.registerPass(PassBuilder: []() { return ScalarEvolutionAnalysis(); });
7753 FAM.registerPass(PassBuilder: []() { return PassInstrumentationAnalysis(); });
7754 TargetIRAnalysis TIRA;
7755 if (TM)
7756 TIRA = TargetIRAnalysis(
7757 [&](const Function &F) { return TM->getTargetTransformInfo(F); });
7758 FAM.registerPass(PassBuilder: [&]() { return TIRA; });
7759
7760 TargetIRAnalysis::Result &&TTI = TIRA.run(F: *F, FAM);
7761 ScalarEvolutionAnalysis SEA;
7762 ScalarEvolution &&SE = SEA.run(F&: *F, AM&: FAM);
7763 DominatorTreeAnalysis DTA;
7764 DominatorTree &&DT = DTA.run(F&: *F, FAM);
7765 LoopAnalysis LIA;
7766 LoopInfo &&LI = LIA.run(F&: *F, AM&: FAM);
7767 AssumptionAnalysis ACT;
7768 AssumptionCache &&AC = ACT.run(F&: *F, FAM);
7769 OptimizationRemarkEmitter ORE{F};
7770
7771 for (Instruction *I : UIs)
7772 I->eraseFromParent();
7773
7774 Loop *L = LI.getLoopFor(BB: CLI->getHeader());
7775 assert(L && "Expecting CanonicalLoopInfo to be recognized as a loop");
7776
7777 TargetTransformInfo::UnrollingPreferences UP = gatherUnrollingPreferences(
7778 L, SE, TTI,
7779 /*BlockFrequencyInfo=*/BFI: nullptr,
7780 /*ProfileSummaryInfo=*/PSI: nullptr, ORE, OptLevel: static_cast<int>(OptLevel),
7781 /*UserThreshold=*/std::nullopt,
7782 /*UserAllowPartial=*/true,
7783 /*UserAllowRuntime=*/UserRuntime: true,
7784 /*UserUpperBound=*/std::nullopt,
7785 /*UserFullUnrollMaxCount=*/std::nullopt);
7786
7787 UP.Force = true;
7788
7789 // Account for additional optimizations taking place before the LoopUnrollPass
7790 // would unroll the loop.
7791 UP.Threshold *= UnrollThresholdFactor;
7792 UP.PartialThreshold *= UnrollThresholdFactor;
7793
7794 // Use normal unroll factors even if the rest of the code is optimized for
7795 // size.
7796 UP.OptSizeThreshold = UP.Threshold;
7797 UP.PartialOptSizeThreshold = UP.PartialThreshold;
7798
7799 LLVM_DEBUG(dbgs() << "Unroll heuristic thresholds:\n"
7800 << " Threshold=" << UP.Threshold << "\n"
7801 << " PartialThreshold=" << UP.PartialThreshold << "\n"
7802 << " OptSizeThreshold=" << UP.OptSizeThreshold << "\n"
7803 << " PartialOptSizeThreshold="
7804 << UP.PartialOptSizeThreshold << "\n");
7805
7806 // Disable peeling.
7807 TargetTransformInfo::PeelingPreferences PP =
7808 gatherPeelingPreferences(L, SE, TTI,
7809 /*UserAllowPeeling=*/false,
7810 /*UserAllowProfileBasedPeeling=*/false,
7811 /*UnrollingSpecficValues=*/false);
7812
7813 SmallPtrSet<const Value *, 32> EphValues;
7814 CodeMetrics::collectEphemeralValues(L, AC: &AC, EphValues);
7815
7816 // Assume that reads and writes to stack variables can be eliminated by
7817 // Mem2Reg, SROA or LICM. That is, don't count them towards the loop body's
7818 // size.
7819 for (BasicBlock *BB : L->blocks()) {
7820 for (Instruction &I : *BB) {
7821 Value *Ptr;
7822 if (auto *Load = dyn_cast<LoadInst>(Val: &I)) {
7823 Ptr = Load->getPointerOperand();
7824 } else if (auto *Store = dyn_cast<StoreInst>(Val: &I)) {
7825 Ptr = Store->getPointerOperand();
7826 } else
7827 continue;
7828
7829 Ptr = Ptr->stripPointerCasts();
7830
7831 if (auto *Alloca = dyn_cast<AllocaInst>(Val: Ptr)) {
7832 if (Alloca->getParent() == &F->getEntryBlock())
7833 EphValues.insert(Ptr: &I);
7834 }
7835 }
7836 }
7837
7838 UnrollCostEstimator UCE(L, TTI, EphValues, UP.BEInsns);
7839
7840 // Loop is not unrollable if the loop contains certain instructions.
7841 if (!UCE.canUnroll()) {
7842 LLVM_DEBUG(dbgs() << "Loop not considered unrollable\n");
7843 return 1;
7844 }
7845
7846 LLVM_DEBUG(dbgs() << "Estimated loop size is " << UCE.getRolledLoopSize()
7847 << "\n");
7848
7849 // TODO: Determine trip count of \p CLI if constant, computeUnrollCount might
7850 // be able to use it.
7851 int TripCount = 0;
7852 int MaxTripCount = 0;
7853 bool MaxOrZero = false;
7854 unsigned TripMultiple = 0;
7855
7856 unsigned Factor =
7857 computeUnrollCount(L, TTI, DT, LI: &LI, AC: &AC, SE, EphValues, ORE: &ORE, TripCount,
7858 MaxTripCount, MaxOrZero, TripMultiple, UCE, UP, PP);
7859 LLVM_DEBUG(dbgs() << "Suggesting unroll factor of " << Factor << "\n");
7860
7861 // This function returns 1 to signal to not unroll a loop.
7862 if (Factor == 0)
7863 return 1;
7864 return Factor;
7865}
7866
7867void OpenMPIRBuilder::unrollLoopPartial(DebugLoc DL, CanonicalLoopInfo *Loop,
7868 int32_t Factor,
7869 CanonicalLoopInfo **UnrolledCLI) {
7870 assert(Factor >= 0 && "Unroll factor must not be negative");
7871
7872 Function *F = Loop->getFunction();
7873 LLVMContext &Ctx = F->getContext();
7874
7875 // If the unrolled loop is not used for another loop-associated directive, it
7876 // is sufficient to add metadata for the LoopUnrollPass.
7877 if (!UnrolledCLI) {
7878 SmallVector<Metadata *, 2> LoopMetadata;
7879 LoopMetadata.push_back(
7880 Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.enable")));
7881
7882 if (Factor >= 1) {
7883 ConstantAsMetadata *FactorConst = ConstantAsMetadata::get(
7884 C: ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: APInt(32, Factor)));
7885 LoopMetadata.push_back(Elt: MDNode::get(
7886 Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.count"), FactorConst}));
7887 }
7888
7889 addLoopMetadata(Loop, Properties: LoopMetadata);
7890 return;
7891 }
7892
7893 // Heuristically determine the unroll factor.
7894 if (Factor == 0)
7895 Factor = computeHeuristicUnrollFactor(CLI: Loop);
7896
7897 // No change required with unroll factor 1.
7898 if (Factor == 1) {
7899 *UnrolledCLI = Loop;
7900 return;
7901 }
7902
7903 assert(Factor >= 2 &&
7904 "unrolling only makes sense with a factor of 2 or larger");
7905
7906 Type *IndVarTy = Loop->getIndVarType();
7907
7908 // Apply partial unrolling by tiling the loop by the unroll-factor, then fully
7909 // unroll the inner loop.
7910 Value *FactorVal =
7911 ConstantInt::get(Ty: IndVarTy, V: APInt(IndVarTy->getIntegerBitWidth(), Factor,
7912 /*isSigned=*/false));
7913 std::vector<CanonicalLoopInfo *> LoopNest =
7914 tileLoops(DL, Loops: {Loop}, TileSizes: {FactorVal});
7915 assert(LoopNest.size() == 2 && "Expect 2 loops after tiling");
7916 *UnrolledCLI = LoopNest[0];
7917 CanonicalLoopInfo *InnerLoop = LoopNest[1];
7918
7919 // LoopUnrollPass can only fully unroll loops with constant trip count.
7920 // Unroll by the unroll factor with a fallback epilog for the remainder
7921 // iterations if necessary.
7922 ConstantAsMetadata *FactorConst = ConstantAsMetadata::get(
7923 C: ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: APInt(32, Factor)));
7924 addLoopMetadata(
7925 Loop: InnerLoop,
7926 Properties: {MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.enable")),
7927 MDNode::get(
7928 Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.count"), FactorConst})});
7929
7930#ifndef NDEBUG
7931 (*UnrolledCLI)->assertOK();
7932#endif
7933}
7934
7935OpenMPIRBuilder::InsertPointTy
7936OpenMPIRBuilder::createCopyPrivate(const LocationDescription &Loc,
7937 llvm::Value *BufSize, llvm::Value *CpyBuf,
7938 llvm::Value *CpyFn, llvm::Value *DidIt) {
7939 if (!updateToLocation(Loc))
7940 return Loc.IP;
7941
7942 uint32_t SrcLocStrSize;
7943 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
7944 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
7945 Value *ThreadId = getOrCreateThreadID(Ident);
7946
7947 llvm::Value *DidItLD = Builder.CreateLoad(Ty: Builder.getInt32Ty(), Ptr: DidIt);
7948
7949 Value *Args[] = {Ident, ThreadId, BufSize, CpyBuf, CpyFn, DidItLD};
7950
7951 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_copyprivate);
7952 createRuntimeFunctionCall(Callee: Fn, Args);
7953
7954 return Builder.saveIP();
7955}
7956
7957OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createSingle(
7958 const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,
7959 FinalizeCallbackTy FiniCB, bool IsNowait, ArrayRef<llvm::Value *> CPVars,
7960 ArrayRef<llvm::Function *> CPFuncs) {
7961
7962 if (!updateToLocation(Loc))
7963 return Loc.IP;
7964
7965 // If needed allocate and initialize `DidIt` with 0.
7966 // DidIt: flag variable: 1=single thread; 0=not single thread.
7967 llvm::Value *DidIt = nullptr;
7968 if (!CPVars.empty()) {
7969 DidIt = Builder.CreateAlloca(Ty: llvm::Type::getInt32Ty(C&: Builder.getContext()));
7970 Builder.CreateStore(Val: Builder.getInt32(C: 0), Ptr: DidIt);
7971 }
7972
7973 Directive OMPD = Directive::OMPD_single;
7974 uint32_t SrcLocStrSize;
7975 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
7976 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
7977 Value *ThreadId = getOrCreateThreadID(Ident);
7978 Value *Args[] = {Ident, ThreadId};
7979
7980 Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_single);
7981 Instruction *EntryCall = createRuntimeFunctionCall(Callee: EntryRTLFn, Args);
7982
7983 Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_end_single);
7984 Instruction *ExitCall = createRuntimeFunctionCall(Callee: ExitRTLFn, Args);
7985
7986 auto FiniCBWrapper = [&](InsertPointTy IP) -> Error {
7987 if (Error Err = FiniCB(IP))
7988 return Err;
7989
7990 // The thread that executes the single region must set `DidIt` to 1.
7991 // This is used by __kmpc_copyprivate, to know if the caller is the
7992 // single thread or not.
7993 if (DidIt)
7994 Builder.CreateStore(Val: Builder.getInt32(C: 1), Ptr: DidIt);
7995
7996 return Error::success();
7997 };
7998
7999 // generates the following:
8000 // if (__kmpc_single()) {
8001 // .... single region ...
8002 // __kmpc_end_single
8003 // }
8004 // __kmpc_copyprivate
8005 // __kmpc_barrier
8006
8007 InsertPointOrErrorTy AfterIP =
8008 EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB: FiniCBWrapper,
8009 /*Conditional*/ true,
8010 /*hasFinalize*/ HasFinalize: true);
8011 if (!AfterIP)
8012 return AfterIP.takeError();
8013
8014 if (DidIt) {
8015 for (size_t I = 0, E = CPVars.size(); I < E; ++I)
8016 // NOTE BufSize is currently unused, so just pass 0.
8017 createCopyPrivate(Loc: LocationDescription(Builder.saveIP(), Loc.DL),
8018 /*BufSize=*/ConstantInt::get(Ty: Int64, V: 0), CpyBuf: CPVars[I],
8019 CpyFn: CPFuncs[I], DidIt);
8020 // NOTE __kmpc_copyprivate already inserts a barrier
8021 } else if (!IsNowait) {
8022 InsertPointOrErrorTy AfterIP =
8023 createBarrier(Loc: LocationDescription(Builder.saveIP(), Loc.DL),
8024 Kind: omp::Directive::OMPD_unknown, /* ForceSimpleCall */ false,
8025 /* CheckCancelFlag */ false);
8026 if (!AfterIP)
8027 return AfterIP.takeError();
8028 }
8029 return Builder.saveIP();
8030}
8031
8032OpenMPIRBuilder::InsertPointOrErrorTy
8033OpenMPIRBuilder::createScope(const LocationDescription &Loc,
8034 BodyGenCallbackTy BodyGenCB,
8035 FinalizeCallbackTy FiniCB, bool IsNowait) {
8036
8037 if (!updateToLocation(Loc))
8038 return Loc.IP;
8039
8040 // All threads execute the scope body — no conditional entry.
8041 InsertPointOrErrorTy AfterIP = EmitOMPInlinedRegion(
8042 OMPD: Directive::OMPD_scope, /*EntryCall=*/nullptr, /*ExitCall=*/nullptr,
8043 BodyGenCB, FiniCB, /*Conditional=*/false, /*HasFinalize=*/true,
8044 /*IsCancellable=*/false);
8045 if (!AfterIP)
8046 return AfterIP.takeError();
8047
8048 Builder.restoreIP(IP: *AfterIP);
8049 if (!IsNowait) {
8050 AfterIP = createBarrier(Loc: LocationDescription(Builder.saveIP(), Loc.DL),
8051 Kind: omp::Directive::OMPD_unknown,
8052 /*ForceSimpleCall=*/false,
8053 /*CheckCancelFlag=*/false);
8054 if (!AfterIP)
8055 return AfterIP.takeError();
8056 }
8057 return Builder.saveIP();
8058}
8059
8060OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createCritical(
8061 const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,
8062 FinalizeCallbackTy FiniCB, StringRef CriticalName, Value *HintInst) {
8063
8064 if (!updateToLocation(Loc))
8065 return Loc.IP;
8066
8067 Directive OMPD = Directive::OMPD_critical;
8068 uint32_t SrcLocStrSize;
8069 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8070 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8071 Value *ThreadId = getOrCreateThreadID(Ident);
8072 Value *LockVar = getOMPCriticalRegionLock(CriticalName);
8073 Value *Args[] = {Ident, ThreadId, LockVar};
8074
8075 SmallVector<llvm::Value *, 4> EnterArgs(std::begin(arr&: Args), std::end(arr&: Args));
8076 Function *RTFn = nullptr;
8077 if (HintInst) {
8078 // Add Hint to entry Args and create call
8079 EnterArgs.push_back(Elt: HintInst);
8080 RTFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_critical_with_hint);
8081 } else {
8082 RTFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_critical);
8083 }
8084 Instruction *EntryCall = createRuntimeFunctionCall(Callee: RTFn, Args: EnterArgs);
8085
8086 Function *ExitRTLFn =
8087 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_end_critical);
8088 Instruction *ExitCall = createRuntimeFunctionCall(Callee: ExitRTLFn, Args);
8089
8090 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
8091 /*Conditional*/ false, /*hasFinalize*/ HasFinalize: true);
8092}
8093
8094OpenMPIRBuilder::InsertPointTy
8095OpenMPIRBuilder::createOrderedDepend(const LocationDescription &Loc,
8096 InsertPointTy AllocaIP, unsigned NumLoops,
8097 ArrayRef<llvm::Value *> StoreValues,
8098 const Twine &Name, bool IsDependSource) {
8099 assert(
8100 llvm::all_of(StoreValues,
8101 [](Value *SV) { return SV->getType()->isIntegerTy(64); }) &&
8102 "OpenMP runtime requires depend vec with i64 type");
8103
8104 if (!updateToLocation(Loc))
8105 return Loc.IP;
8106
8107 // Allocate space for vector and generate alloc instruction.
8108 auto *ArrI64Ty = ArrayType::get(ElementType: Int64, NumElements: NumLoops);
8109 Builder.restoreIP(IP: AllocaIP);
8110 AllocaInst *ArgsBase = Builder.CreateAlloca(Ty: ArrI64Ty, ArraySize: nullptr, Name);
8111 ArgsBase->setAlignment(Align(8));
8112 updateToLocation(Loc);
8113
8114 // Store the index value with offset in depend vector.
8115 for (unsigned I = 0; I < NumLoops; ++I) {
8116 Value *DependAddrGEPIter = Builder.CreateInBoundsGEP(
8117 Ty: ArrI64Ty, Ptr: ArgsBase, IdxList: {Builder.getInt64(C: 0), Builder.getInt64(C: I)});
8118 StoreInst *STInst = Builder.CreateStore(Val: StoreValues[I], Ptr: DependAddrGEPIter);
8119 STInst->setAlignment(Align(8));
8120 }
8121
8122 Value *DependBaseAddrGEP = Builder.CreateInBoundsGEP(
8123 Ty: ArrI64Ty, Ptr: ArgsBase, IdxList: {Builder.getInt64(C: 0), Builder.getInt64(C: 0)});
8124
8125 uint32_t SrcLocStrSize;
8126 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8127 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8128 Value *ThreadId = getOrCreateThreadID(Ident);
8129 Value *Args[] = {Ident, ThreadId, DependBaseAddrGEP};
8130
8131 Function *RTLFn = nullptr;
8132 if (IsDependSource)
8133 RTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_doacross_post);
8134 else
8135 RTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_doacross_wait);
8136 createRuntimeFunctionCall(Callee: RTLFn, Args);
8137
8138 return Builder.saveIP();
8139}
8140
8141OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createOrderedThreadsSimd(
8142 const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,
8143 FinalizeCallbackTy FiniCB, bool IsThreads) {
8144 if (!updateToLocation(Loc))
8145 return Loc.IP;
8146
8147 Directive OMPD = Directive::OMPD_ordered_blockassoc;
8148 Instruction *EntryCall = nullptr;
8149 Instruction *ExitCall = nullptr;
8150
8151 if (IsThreads) {
8152 uint32_t SrcLocStrSize;
8153 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8154 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8155 Value *ThreadId = getOrCreateThreadID(Ident);
8156 Value *Args[] = {Ident, ThreadId};
8157
8158 Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_ordered);
8159 EntryCall = createRuntimeFunctionCall(Callee: EntryRTLFn, Args);
8160
8161 Function *ExitRTLFn =
8162 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_end_ordered);
8163 ExitCall = createRuntimeFunctionCall(Callee: ExitRTLFn, Args);
8164 }
8165
8166 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
8167 /*Conditional*/ false, /*hasFinalize*/ HasFinalize: true);
8168}
8169
8170OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::EmitOMPInlinedRegion(
8171 Directive OMPD, Instruction *EntryCall, Instruction *ExitCall,
8172 BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB, bool Conditional,
8173 bool HasFinalize, bool IsCancellable) {
8174
8175 if (HasFinalize)
8176 FinalizationStack.push_back(Elt: {FiniCB, OMPD, IsCancellable});
8177
8178 // Create inlined region's entry and body blocks, in preparation
8179 // for conditional creation
8180 BasicBlock *EntryBB = Builder.GetInsertBlock();
8181 Instruction *SplitPos = EntryBB->getTerminatorOrNull();
8182 if (!isa_and_nonnull<UncondBrInst, CondBrInst>(Val: SplitPos))
8183 SplitPos = new UnreachableInst(Builder.getContext(), EntryBB);
8184 BasicBlock *ExitBB = EntryBB->splitBasicBlock(I: SplitPos, BBName: "omp_region.end");
8185 BasicBlock *FiniBB =
8186 EntryBB->splitBasicBlock(I: EntryBB->getTerminator(), BBName: "omp_region.finalize");
8187
8188 Builder.SetInsertPoint(EntryBB->getTerminator());
8189 emitCommonDirectiveEntry(OMPD, EntryCall, ExitBB, Conditional);
8190
8191 // generate body
8192 if (Error Err =
8193 BodyGenCB(/* AllocaIP */ InsertPointTy(),
8194 /* CodeGenIP */ Builder.saveIP(), /* DeallocBlocks */ {}))
8195 return Err;
8196
8197 // emit exit call and do any needed finalization.
8198 auto FinIP = InsertPointTy(FiniBB, FiniBB->getFirstInsertionPt());
8199 assert(FiniBB->getTerminator()->getNumSuccessors() == 1 &&
8200 FiniBB->getTerminator()->getSuccessor(0) == ExitBB &&
8201 "Unexpected control flow graph state!!");
8202 InsertPointOrErrorTy AfterIP =
8203 emitCommonDirectiveExit(OMPD, FinIP, ExitCall, HasFinalize);
8204 if (!AfterIP)
8205 return AfterIP.takeError();
8206
8207 // If we are skipping the region of a non conditional, remove the exit
8208 // block, and clear the builder's insertion point.
8209 assert(SplitPos->getParent() == ExitBB &&
8210 "Unexpected Insertion point location!");
8211 auto merged = MergeBlockIntoPredecessor(BB: ExitBB);
8212 BasicBlock *ExitPredBB = SplitPos->getParent();
8213 auto InsertBB = merged ? ExitPredBB : ExitBB;
8214 if (!isa_and_nonnull<UncondBrInst, CondBrInst>(Val: SplitPos))
8215 SplitPos->eraseFromParent();
8216 Builder.SetInsertPoint(InsertBB);
8217
8218 return Builder.saveIP();
8219}
8220
8221OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitCommonDirectiveEntry(
8222 Directive OMPD, Value *EntryCall, BasicBlock *ExitBB, bool Conditional) {
8223 // if nothing to do, Return current insertion point.
8224 if (!Conditional || !EntryCall)
8225 return Builder.saveIP();
8226
8227 BasicBlock *EntryBB = Builder.GetInsertBlock();
8228 Value *CallBool = Builder.CreateIsNotNull(Arg: EntryCall);
8229 auto *ThenBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp_region.body");
8230 auto *UI = new UnreachableInst(Builder.getContext(), ThenBB);
8231
8232 // Emit thenBB and set the Builder's insertion point there for
8233 // body generation next. Place the block after the current block.
8234 Function *CurFn = EntryBB->getParent();
8235 CurFn->insert(Position: std::next(x: EntryBB->getIterator()), BB: ThenBB);
8236
8237 // Move Entry branch to end of ThenBB, and replace with conditional
8238 // branch (If-stmt)
8239 Instruction *EntryBBTI = EntryBB->getTerminator();
8240 Builder.CreateCondBr(Cond: CallBool, True: ThenBB, False: ExitBB);
8241 EntryBBTI->removeFromParent();
8242 Builder.SetInsertPoint(UI);
8243 Builder.Insert(I: EntryBBTI);
8244 UI->eraseFromParent();
8245 Builder.SetInsertPoint(ThenBB->getTerminator());
8246
8247 // return an insertion point to ExitBB.
8248 return IRBuilder<>::InsertPoint(ExitBB, ExitBB->getFirstInsertionPt());
8249}
8250
8251OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitCommonDirectiveExit(
8252 omp::Directive OMPD, InsertPointTy FinIP, Instruction *ExitCall,
8253 bool HasFinalize) {
8254
8255 Builder.restoreIP(IP: FinIP);
8256
8257 // If there is finalization to do, emit it before the exit call
8258 if (HasFinalize) {
8259 assert(!FinalizationStack.empty() &&
8260 "Unexpected finalization stack state!");
8261
8262 FinalizationInfo Fi = FinalizationStack.pop_back_val();
8263 assert(Fi.DK == OMPD && "Unexpected Directive for Finalization call!");
8264
8265 if (Error Err = Fi.mergeFiniBB(Builder, OtherFiniBB: FinIP.getBlock()))
8266 return std::move(Err);
8267
8268 // Exit condition: insertion point is before the terminator of the new Fini
8269 // block
8270 Builder.SetInsertPoint(FinIP.getBlock()->getTerminator());
8271 }
8272
8273 if (!ExitCall)
8274 return Builder.saveIP();
8275
8276 // place the Exitcall as last instruction before Finalization block terminator
8277 ExitCall->removeFromParent();
8278 Builder.Insert(I: ExitCall);
8279
8280 return IRBuilder<>::InsertPoint(ExitCall->getParent(),
8281 ExitCall->getIterator());
8282}
8283
8284OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createCopyinClauseBlocks(
8285 InsertPointTy IP, Value *MasterAddr, Value *PrivateAddr,
8286 llvm::IntegerType *IntPtrTy, bool BranchtoEnd) {
8287 if (!IP.isSet())
8288 return IP;
8289
8290 IRBuilder<>::InsertPointGuard IPG(Builder);
8291
8292 // creates the following CFG structure
8293 // OMP_Entry : (MasterAddr != PrivateAddr)?
8294 // F T
8295 // | \
8296 // | copin.not.master
8297 // | /
8298 // v /
8299 // copyin.not.master.end
8300 // |
8301 // v
8302 // OMP.Entry.Next
8303
8304 BasicBlock *OMP_Entry = IP.getBlock();
8305 Function *CurFn = OMP_Entry->getParent();
8306 BasicBlock *CopyBegin =
8307 BasicBlock::Create(Context&: M.getContext(), Name: "copyin.not.master", Parent: CurFn);
8308 BasicBlock *CopyEnd = nullptr;
8309
8310 // If entry block is terminated, split to preserve the branch to following
8311 // basic block (i.e. OMP.Entry.Next), otherwise, leave everything as is.
8312 if (isa_and_nonnull<CondBrInst>(Val: OMP_Entry->getTerminatorOrNull())) {
8313 CopyEnd = OMP_Entry->splitBasicBlock(I: OMP_Entry->getTerminator(),
8314 BBName: "copyin.not.master.end");
8315 OMP_Entry->getTerminator()->eraseFromParent();
8316 } else {
8317 CopyEnd =
8318 BasicBlock::Create(Context&: M.getContext(), Name: "copyin.not.master.end", Parent: CurFn);
8319 }
8320
8321 Builder.SetInsertPoint(OMP_Entry);
8322 Value *MasterPtr = Builder.CreatePtrToInt(V: MasterAddr, DestTy: IntPtrTy);
8323 Value *PrivatePtr = Builder.CreatePtrToInt(V: PrivateAddr, DestTy: IntPtrTy);
8324 Value *cmp = Builder.CreateICmpNE(LHS: MasterPtr, RHS: PrivatePtr);
8325 Builder.CreateCondBr(Cond: cmp, True: CopyBegin, False: CopyEnd);
8326
8327 Builder.SetInsertPoint(CopyBegin);
8328 if (BranchtoEnd)
8329 Builder.SetInsertPoint(Builder.CreateBr(Dest: CopyEnd));
8330
8331 return Builder.saveIP();
8332}
8333
8334CallInst *OpenMPIRBuilder::createOMPAlloc(const LocationDescription &Loc,
8335 Value *Size, Value *Allocator,
8336 std::string Name) {
8337 IRBuilder<>::InsertPointGuard IPG(Builder);
8338 if (!updateToLocation(Loc))
8339 return nullptr;
8340
8341 uint32_t SrcLocStrSize;
8342 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8343 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8344 Value *ThreadId = getOrCreateThreadID(Ident);
8345 Value *Args[] = {ThreadId, Size, Allocator};
8346
8347 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_alloc);
8348
8349 return createRuntimeFunctionCall(Callee: Fn, Args, Name);
8350}
8351
8352CallInst *OpenMPIRBuilder::createOMPAlignedAlloc(const LocationDescription &Loc,
8353 Value *Align, Value *Size,
8354 Value *Allocator,
8355 std::string Name) {
8356 IRBuilder<>::InsertPointGuard IPG(Builder);
8357 if (!updateToLocation(Loc))
8358 return nullptr;
8359
8360 uint32_t SrcLocStrSize;
8361 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8362 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8363 Value *ThreadId = getOrCreateThreadID(Ident);
8364 Value *Args[] = {ThreadId, Align, Size, Allocator};
8365
8366 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_aligned_alloc);
8367
8368 return Builder.CreateCall(Callee: Fn, Args, Name);
8369}
8370
8371CallInst *OpenMPIRBuilder::createOMPFree(const LocationDescription &Loc,
8372 Value *Addr, Value *Allocator,
8373 std::string Name) {
8374 IRBuilder<>::InsertPointGuard IPG(Builder);
8375 if (!updateToLocation(Loc))
8376 return nullptr;
8377
8378 uint32_t SrcLocStrSize;
8379 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8380 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8381 Value *ThreadId = getOrCreateThreadID(Ident);
8382 Value *Args[] = {ThreadId, Addr, Allocator};
8383 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_free);
8384 return createRuntimeFunctionCall(Callee: Fn, Args, Name);
8385}
8386
8387CallInst *OpenMPIRBuilder::createOMPAllocShared(const LocationDescription &Loc,
8388 Value *Size,
8389 const Twine &Name) {
8390 IRBuilder<>::InsertPointGuard IPG(Builder);
8391 updateToLocation(Loc);
8392
8393 Value *Args[] = {Size};
8394 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_alloc_shared);
8395 CallInst *Call = Builder.CreateCall(Callee: Fn, Args, Name);
8396 Call->addRetAttr(Attr: Attribute::getWithAlignment(
8397 Context&: M.getContext(), Alignment: M.getDataLayout().getPrefTypeAlign(Ty: Int64)));
8398 return Call;
8399}
8400
8401CallInst *OpenMPIRBuilder::createOMPAllocShared(const LocationDescription &Loc,
8402 Type *VarType,
8403 const Twine &Name) {
8404 return createOMPAllocShared(
8405 Loc, Size: Builder.getInt64(C: M.getDataLayout().getTypeAllocSize(Ty: VarType)), Name);
8406}
8407
8408CallInst *OpenMPIRBuilder::createOMPFreeShared(const LocationDescription &Loc,
8409 Value *Addr, Value *Size,
8410 const Twine &Name) {
8411 IRBuilder<>::InsertPointGuard IPG(Builder);
8412 updateToLocation(Loc);
8413
8414 Value *Args[] = {Addr, Size};
8415 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_free_shared);
8416 return Builder.CreateCall(Callee: Fn, Args, Name);
8417}
8418
8419CallInst *OpenMPIRBuilder::createOMPFreeShared(const LocationDescription &Loc,
8420 Value *Addr, Type *VarType,
8421 const Twine &Name) {
8422 return createOMPFreeShared(
8423 Loc, Addr, Size: Builder.getInt64(C: M.getDataLayout().getTypeAllocSize(Ty: VarType)),
8424 Name);
8425}
8426
8427CallInst *OpenMPIRBuilder::createOMPInteropInit(
8428 const LocationDescription &Loc, Value *InteropVar,
8429 omp::OMPInteropType InteropType, Value *Device, Value *NumDependences,
8430 Value *DependenceAddress, bool HaveNowaitClause) {
8431 IRBuilder<>::InsertPointGuard IPG(Builder);
8432 updateToLocation(Loc);
8433
8434 uint32_t SrcLocStrSize;
8435 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8436 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8437 Value *ThreadId = getOrCreateThreadID(Ident);
8438 if (Device == nullptr)
8439 Device = Constant::getAllOnesValue(Ty: Int32);
8440 else if (Device->getType() != Int32)
8441 Device = Builder.CreateIntCast(V: Device, DestTy: Int32, /*isSigned=*/true);
8442 Constant *InteropTypeVal = ConstantInt::get(Ty: Int32, V: (int)InteropType);
8443 if (NumDependences == nullptr) {
8444 NumDependences = ConstantInt::get(Ty: Int32, V: 0);
8445 PointerType *PointerTypeVar = PointerType::getUnqual(C&: M.getContext());
8446 DependenceAddress = ConstantPointerNull::get(T: PointerTypeVar);
8447 }
8448 Value *HaveNowaitClauseVal = ConstantInt::get(Ty: Int32, V: HaveNowaitClause);
8449 Value *Args[] = {
8450 Ident, ThreadId, InteropVar, InteropTypeVal,
8451 Device, NumDependences, DependenceAddress, HaveNowaitClauseVal};
8452
8453 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___tgt_interop_init);
8454
8455 return createRuntimeFunctionCall(Callee: Fn, Args);
8456}
8457
8458CallInst *OpenMPIRBuilder::createOMPInteropDestroy(
8459 const LocationDescription &Loc, Value *InteropVar, Value *Device,
8460 Value *NumDependences, Value *DependenceAddress, bool HaveNowaitClause) {
8461 IRBuilder<>::InsertPointGuard IPG(Builder);
8462 updateToLocation(Loc);
8463
8464 uint32_t SrcLocStrSize;
8465 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8466 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8467 Value *ThreadId = getOrCreateThreadID(Ident);
8468 if (Device == nullptr)
8469 Device = Constant::getAllOnesValue(Ty: Int32);
8470 else if (Device->getType() != Int32)
8471 Device = Builder.CreateIntCast(V: Device, DestTy: Int32, /*isSigned=*/true);
8472 if (NumDependences == nullptr) {
8473 NumDependences = ConstantInt::get(Ty: Int32, V: 0);
8474 PointerType *PointerTypeVar = PointerType::getUnqual(C&: M.getContext());
8475 DependenceAddress = ConstantPointerNull::get(T: PointerTypeVar);
8476 }
8477 Value *HaveNowaitClauseVal = ConstantInt::get(Ty: Int32, V: HaveNowaitClause);
8478 Value *Args[] = {
8479 Ident, ThreadId, InteropVar, Device,
8480 NumDependences, DependenceAddress, HaveNowaitClauseVal};
8481
8482 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___tgt_interop_destroy);
8483
8484 return createRuntimeFunctionCall(Callee: Fn, Args);
8485}
8486
8487CallInst *OpenMPIRBuilder::createOMPInteropUse(const LocationDescription &Loc,
8488 Value *InteropVar, Value *Device,
8489 Value *NumDependences,
8490 Value *DependenceAddress,
8491 bool HaveNowaitClause) {
8492 IRBuilder<>::InsertPointGuard IPG(Builder);
8493 updateToLocation(Loc);
8494 uint32_t SrcLocStrSize;
8495 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8496 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8497 Value *ThreadId = getOrCreateThreadID(Ident);
8498 if (Device == nullptr)
8499 Device = Constant::getAllOnesValue(Ty: Int32);
8500 else if (Device->getType() != Int32)
8501 Device = Builder.CreateIntCast(V: Device, DestTy: Int32, /*isSigned=*/true);
8502 if (NumDependences == nullptr) {
8503 NumDependences = ConstantInt::get(Ty: Int32, V: 0);
8504 PointerType *PointerTypeVar = PointerType::getUnqual(C&: M.getContext());
8505 DependenceAddress = ConstantPointerNull::get(T: PointerTypeVar);
8506 }
8507 Value *HaveNowaitClauseVal = ConstantInt::get(Ty: Int32, V: HaveNowaitClause);
8508 Value *Args[] = {
8509 Ident, ThreadId, InteropVar, Device,
8510 NumDependences, DependenceAddress, HaveNowaitClauseVal};
8511
8512 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___tgt_interop_use);
8513
8514 return createRuntimeFunctionCall(Callee: Fn, Args);
8515}
8516
8517CallInst *OpenMPIRBuilder::createCachedThreadPrivate(
8518 const LocationDescription &Loc, llvm::Value *Pointer,
8519 llvm::ConstantInt *Size, const llvm::Twine &Name) {
8520 IRBuilder<>::InsertPointGuard IPG(Builder);
8521 updateToLocation(Loc);
8522
8523 uint32_t SrcLocStrSize;
8524 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8525 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8526 Value *ThreadId = getOrCreateThreadID(Ident);
8527 Constant *ThreadPrivateCache =
8528 getOrCreateInternalVariable(Ty: Int8PtrPtr, Name: Name.str());
8529 llvm::Value *Args[] = {Ident, ThreadId, Pointer, Size, ThreadPrivateCache};
8530
8531 Function *Fn =
8532 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_threadprivate_cached);
8533
8534 return createRuntimeFunctionCall(Callee: Fn, Args);
8535}
8536
8537OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTargetInit(
8538 const LocationDescription &Loc,
8539 const llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs &Attrs) {
8540 assert(!Attrs.MaxThreads.empty() && !Attrs.MaxTeams.empty() &&
8541 "expected num_threads and num_teams to be specified");
8542
8543 if (!updateToLocation(Loc))
8544 return Loc.IP;
8545
8546 uint32_t SrcLocStrSize;
8547 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8548 Constant *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8549 Constant *IsSPMDVal = ConstantInt::getSigned(Ty: Int8, V: Attrs.ExecFlags);
8550 Constant *UseGenericStateMachineVal = ConstantInt::getSigned(
8551 Ty: Int8, V: Attrs.ExecFlags != omp::OMP_TGT_EXEC_MODE_SPMD &&
8552 Attrs.ExecFlags != omp::OMP_TGT_EXEC_MODE_SPMD_NO_LOOP);
8553 Constant *MayUseNestedParallelismVal = ConstantInt::getSigned(Ty: Int8, V: true);
8554 Constant *DebugIndentionLevelVal = ConstantInt::getSigned(Ty: Int16, V: 0);
8555
8556 Function *DebugKernelWrapper = Builder.GetInsertBlock()->getParent();
8557 Function *Kernel = DebugKernelWrapper;
8558
8559 // We need to strip the debug prefix to get the correct kernel name.
8560 StringRef KernelName = Kernel->getName();
8561 const std::string DebugPrefix = "_debug__";
8562 if (KernelName.ends_with(Suffix: DebugPrefix)) {
8563 KernelName = KernelName.drop_back(N: DebugPrefix.length());
8564 Kernel = M.getFunction(Name: KernelName);
8565 assert(Kernel && "Expected the real kernel to exist");
8566 }
8567
8568 // Manifest the launch configuration in the metadata matching the kernel
8569 // environment.
8570 if (Attrs.MinTeams.front() > 1 || Attrs.MaxTeams.front() > 0)
8571 writeTeamsForKernel(T, Kernel&: *Kernel, LB: Attrs.MinTeams.front(),
8572 UB: Attrs.MaxTeams.front());
8573
8574 // If MaxThreads is not set and needs adjustment, select the maximum between
8575 // the default workgroup size and the MinThreads value.
8576 int32_t MaxThreadsVal = Attrs.MaxThreads.front();
8577 if (MaxThreadsVal < 0 && UseDefaultMaxThreads) {
8578 if (hasGridValue(T)) {
8579 MaxThreadsVal =
8580 std::max(a: int32_t(getGridValue(T, Kernel).GV_Default_WG_Size),
8581 b: Attrs.MinThreads.front());
8582 } else {
8583 MaxThreadsVal = Attrs.MinThreads.front();
8584 }
8585 }
8586
8587 if (MaxThreadsVal > 0)
8588 writeThreadBoundsForKernel(T, Kernel&: *Kernel, LB: Attrs.MinThreads.front(),
8589 UB: MaxThreadsVal);
8590
8591 Constant *MinThreads =
8592 ConstantInt::getSigned(Ty: Int32, V: Attrs.MinThreads.front());
8593 Constant *MaxThreads = ConstantInt::getSigned(Ty: Int32, V: MaxThreadsVal);
8594 Constant *MinTeams = ConstantInt::getSigned(Ty: Int32, V: Attrs.MinTeams.front());
8595 Constant *MaxTeams = ConstantInt::getSigned(Ty: Int32, V: Attrs.MaxTeams.front());
8596 Constant *ReductionDataSize =
8597 ConstantInt::getSigned(Ty: Int32, V: Attrs.ReductionDataSize);
8598
8599 Function *Fn = getOrCreateRuntimeFunctionPtr(
8600 FnID: omp::RuntimeFunction::OMPRTL___kmpc_target_init);
8601 const DataLayout &DL = Fn->getDataLayout();
8602
8603 Twine DynamicEnvironmentName = KernelName + "_dynamic_environment";
8604 Constant *DynamicEnvironmentInitializer =
8605 ConstantStruct::get(T: DynamicEnvironment, V: {DebugIndentionLevelVal});
8606 GlobalVariable *DynamicEnvironmentGV = new GlobalVariable(
8607 M, DynamicEnvironment, /*IsConstant=*/false, GlobalValue::WeakODRLinkage,
8608 DynamicEnvironmentInitializer, DynamicEnvironmentName,
8609 /*InsertBefore=*/nullptr, GlobalValue::NotThreadLocal,
8610 DL.getDefaultGlobalsAddressSpace());
8611 DynamicEnvironmentGV->setVisibility(GlobalValue::ProtectedVisibility);
8612
8613 Constant *DynamicEnvironment =
8614 DynamicEnvironmentGV->getType() == DynamicEnvironmentPtr
8615 ? DynamicEnvironmentGV
8616 : ConstantExpr::getAddrSpaceCast(C: DynamicEnvironmentGV,
8617 Ty: DynamicEnvironmentPtr);
8618
8619 Constant *ConfigurationEnvironmentInitializer = ConstantStruct::get(
8620 T: ConfigurationEnvironment, V: {
8621 UseGenericStateMachineVal,
8622 MayUseNestedParallelismVal,
8623 IsSPMDVal,
8624 MinThreads,
8625 MaxThreads,
8626 MinTeams,
8627 MaxTeams,
8628 ReductionDataSize,
8629 });
8630 Constant *KernelEnvironmentInitializer = ConstantStruct::get(
8631 T: KernelEnvironment, V: {
8632 ConfigurationEnvironmentInitializer,
8633 Ident,
8634 DynamicEnvironment,
8635 });
8636 std::string KernelEnvironmentName =
8637 (KernelName + "_kernel_environment").str();
8638 GlobalVariable *KernelEnvironmentGV = new GlobalVariable(
8639 M, KernelEnvironment, /*IsConstant=*/true, GlobalValue::WeakODRLinkage,
8640 KernelEnvironmentInitializer, KernelEnvironmentName,
8641 /*InsertBefore=*/nullptr, GlobalValue::NotThreadLocal,
8642 DL.getDefaultGlobalsAddressSpace());
8643 KernelEnvironmentGV->setVisibility(GlobalValue::ProtectedVisibility);
8644
8645 Constant *KernelEnvironment =
8646 KernelEnvironmentGV->getType() == KernelEnvironmentPtr
8647 ? KernelEnvironmentGV
8648 : ConstantExpr::getAddrSpaceCast(C: KernelEnvironmentGV,
8649 Ty: KernelEnvironmentPtr);
8650 Value *KernelLaunchEnvironment =
8651 DebugKernelWrapper->getArg(i: DebugKernelWrapper->arg_size() - 1);
8652 Type *KernelLaunchEnvParamTy = Fn->getFunctionType()->getParamType(i: 1);
8653 KernelLaunchEnvironment =
8654 KernelLaunchEnvironment->getType() == KernelLaunchEnvParamTy
8655 ? KernelLaunchEnvironment
8656 : Builder.CreateAddrSpaceCast(V: KernelLaunchEnvironment,
8657 DestTy: KernelLaunchEnvParamTy);
8658 CallInst *ThreadKind = createRuntimeFunctionCall(
8659 Callee: Fn, Args: {KernelEnvironment, KernelLaunchEnvironment});
8660
8661 Value *ExecUserCode = Builder.CreateICmpEQ(
8662 LHS: ThreadKind, RHS: Constant::getAllOnesValue(Ty: ThreadKind->getType()),
8663 Name: "exec_user_code");
8664
8665 // ThreadKind = __kmpc_target_init(...)
8666 // if (ThreadKind == -1)
8667 // user_code
8668 // else
8669 // return;
8670
8671 auto *UI = Builder.CreateUnreachable();
8672 BasicBlock *CheckBB = UI->getParent();
8673 BasicBlock *UserCodeEntryBB = CheckBB->splitBasicBlock(I: UI, BBName: "user_code.entry");
8674
8675 BasicBlock *WorkerExitBB = BasicBlock::Create(
8676 Context&: CheckBB->getContext(), Name: "worker.exit", Parent: CheckBB->getParent());
8677 Builder.SetInsertPoint(WorkerExitBB);
8678 Builder.CreateRetVoid();
8679
8680 auto *CheckBBTI = CheckBB->getTerminator();
8681 Builder.SetInsertPoint(CheckBBTI);
8682 Builder.CreateCondBr(Cond: ExecUserCode, True: UI->getParent(), False: WorkerExitBB);
8683
8684 CheckBBTI->eraseFromParent();
8685 UI->eraseFromParent();
8686
8687 // Continue in the "user_code" block, see diagram above and in
8688 // openmp/libomptarget/deviceRTLs/common/include/target.h .
8689 return InsertPointTy(UserCodeEntryBB, UserCodeEntryBB->getFirstInsertionPt());
8690}
8691
8692void OpenMPIRBuilder::createTargetDeinit(const LocationDescription &Loc,
8693 int32_t TeamsReductionDataSize) {
8694 if (!updateToLocation(Loc))
8695 return;
8696
8697 Function *Fn = getOrCreateRuntimeFunctionPtr(
8698 FnID: omp::RuntimeFunction::OMPRTL___kmpc_target_deinit);
8699
8700 createRuntimeFunctionCall(Callee: Fn, Args: {});
8701
8702 if (!TeamsReductionDataSize)
8703 return;
8704
8705 Function *Kernel = Builder.GetInsertBlock()->getParent();
8706 // We need to strip the debug prefix to get the correct kernel name.
8707 StringRef KernelName = Kernel->getName();
8708 const std::string DebugPrefix = "_debug__";
8709 if (KernelName.ends_with(Suffix: DebugPrefix))
8710 KernelName = KernelName.drop_back(N: DebugPrefix.length());
8711 auto *KernelEnvironmentGV =
8712 M.getNamedGlobal(Name: (KernelName + "_kernel_environment").str());
8713 assert(KernelEnvironmentGV && "Expected kernel environment global\n");
8714 auto *KernelEnvironmentInitializer = KernelEnvironmentGV->getInitializer();
8715 auto *NewInitializer = ConstantFoldInsertValueInstruction(
8716 Agg: KernelEnvironmentInitializer,
8717 Val: ConstantInt::get(Ty: Int32, V: TeamsReductionDataSize), Idxs: {0, 7});
8718 KernelEnvironmentGV->setInitializer(NewInitializer);
8719}
8720
8721static void updateNVPTXAttr(Function &Kernel, StringRef Name, int32_t Value,
8722 bool Min) {
8723 if (Kernel.hasFnAttribute(Kind: Name)) {
8724 int32_t OldLimit = Kernel.getFnAttributeAsParsedInteger(Kind: Name);
8725 Value = Min ? std::min(a: OldLimit, b: Value) : std::max(a: OldLimit, b: Value);
8726 }
8727 Kernel.addFnAttr(Kind: Name, Val: llvm::utostr(X: Value));
8728}
8729
8730std::pair<int32_t, int32_t>
8731OpenMPIRBuilder::readThreadBoundsForKernel(const Triple &T, Function &Kernel) {
8732 int32_t ThreadLimit =
8733 Kernel.getFnAttributeAsParsedInteger(Kind: "omp_target_thread_limit");
8734
8735 if (T.isAMDGPU()) {
8736 const auto &Attr = Kernel.getFnAttribute(Kind: "amdgpu-flat-work-group-size");
8737 if (!Attr.isValid() || !Attr.isStringAttribute())
8738 return {0, ThreadLimit};
8739 auto [LBStr, UBStr] = Attr.getValueAsString().split(Separator: ',');
8740 int32_t LB, UB;
8741 if (!llvm::to_integer(S: UBStr, Num&: UB, Base: 10))
8742 return {0, ThreadLimit};
8743 UB = ThreadLimit ? std::min(a: ThreadLimit, b: UB) : UB;
8744 if (!llvm::to_integer(S: LBStr, Num&: LB, Base: 10))
8745 return {0, UB};
8746 return {LB, UB};
8747 }
8748
8749 if (Kernel.hasFnAttribute(Kind: NVVMAttr::MaxNTID)) {
8750 int32_t UB = Kernel.getFnAttributeAsParsedInteger(Kind: NVVMAttr::MaxNTID);
8751 return {0, ThreadLimit ? std::min(a: ThreadLimit, b: UB) : UB};
8752 }
8753 return {0, ThreadLimit};
8754}
8755
8756void OpenMPIRBuilder::writeThreadBoundsForKernel(const Triple &T,
8757 Function &Kernel, int32_t LB,
8758 int32_t UB) {
8759 Kernel.addFnAttr(Kind: "omp_target_thread_limit", Val: std::to_string(val: UB));
8760
8761 if (T.isAMDGPU()) {
8762 Kernel.addFnAttr(Kind: "amdgpu-flat-work-group-size",
8763 Val: llvm::utostr(X: LB) + "," + llvm::utostr(X: UB));
8764 return;
8765 }
8766
8767 updateNVPTXAttr(Kernel, Name: NVVMAttr::MaxNTID, Value: UB, Min: true);
8768}
8769
8770std::pair<int32_t, int32_t>
8771OpenMPIRBuilder::readTeamBoundsForKernel(const Triple &, Function &Kernel) {
8772 // TODO: Read from backend annotations if available.
8773 return {0, Kernel.getFnAttributeAsParsedInteger(Kind: "omp_target_num_teams")};
8774}
8775
8776void OpenMPIRBuilder::writeTeamsForKernel(const Triple &T, Function &Kernel,
8777 int32_t LB, int32_t UB) {
8778 if (UB > 0) {
8779 if (T.isNVPTX())
8780 Kernel.addFnAttr(Kind: NVVMAttr::MaxClusterRank, Val: llvm::utostr(X: UB));
8781 if (T.isAMDGPU())
8782 Kernel.addFnAttr(Kind: "amdgpu-max-num-workgroups", Val: llvm::utostr(X: UB) + ",1,1");
8783 }
8784
8785 Kernel.addFnAttr(Kind: "omp_target_num_teams", Val: std::to_string(val: LB));
8786}
8787
8788void OpenMPIRBuilder::setOutlinedTargetRegionFunctionAttributes(
8789 Function *OutlinedFn) {
8790 if (Config.isTargetDevice()) {
8791 OutlinedFn->setLinkage(GlobalValue::WeakODRLinkage);
8792 // TODO: Determine if DSO local can be set to true.
8793 OutlinedFn->setDSOLocal(false);
8794 OutlinedFn->setVisibility(GlobalValue::ProtectedVisibility);
8795 if (T.isAMDGCN())
8796 OutlinedFn->setCallingConv(CallingConv::AMDGPU_KERNEL);
8797 else if (T.isNVPTX())
8798 OutlinedFn->setCallingConv(CallingConv::PTX_Kernel);
8799 else if (T.isSPIRV())
8800 OutlinedFn->setCallingConv(CallingConv::SPIR_KERNEL);
8801 }
8802}
8803
8804Constant *OpenMPIRBuilder::createOutlinedFunctionID(Function *OutlinedFn,
8805 StringRef EntryFnIDName) {
8806 if (Config.isTargetDevice()) {
8807 assert(OutlinedFn && "The outlined function must exist if embedded");
8808 return OutlinedFn;
8809 }
8810
8811 return new GlobalVariable(
8812 M, Builder.getInt8Ty(), /*isConstant=*/true, GlobalValue::WeakAnyLinkage,
8813 Constant::getNullValue(Ty: Builder.getInt8Ty()), EntryFnIDName);
8814}
8815
8816Constant *OpenMPIRBuilder::createTargetRegionEntryAddr(Function *OutlinedFn,
8817 StringRef EntryFnName) {
8818 if (OutlinedFn)
8819 return OutlinedFn;
8820
8821 assert(!M.getGlobalVariable(EntryFnName, true) &&
8822 "Named kernel already exists?");
8823 return new GlobalVariable(
8824 M, Builder.getInt8Ty(), /*isConstant=*/true, GlobalValue::InternalLinkage,
8825 Constant::getNullValue(Ty: Builder.getInt8Ty()), EntryFnName);
8826}
8827
8828Error OpenMPIRBuilder::emitTargetRegionFunction(
8829 TargetRegionEntryInfo &EntryInfo,
8830 FunctionGenCallback &GenerateFunctionCallback, bool IsOffloadEntry,
8831 Function *&OutlinedFn, Constant *&OutlinedFnID) {
8832
8833 SmallString<64> EntryFnName;
8834 OffloadInfoManager.getTargetRegionEntryFnName(Name&: EntryFnName, EntryInfo);
8835
8836 if (Config.isTargetDevice() || !Config.openMPOffloadMandatory()) {
8837 Expected<Function *> CBResult = GenerateFunctionCallback(EntryFnName);
8838 if (!CBResult)
8839 return CBResult.takeError();
8840 OutlinedFn = *CBResult;
8841 } else {
8842 OutlinedFn = nullptr;
8843 }
8844
8845 // If this target outline function is not an offload entry, we don't need to
8846 // register it. This may be in the case of a false if clause, or if there are
8847 // no OpenMP targets.
8848 if (!IsOffloadEntry)
8849 return Error::success();
8850
8851 std::string EntryFnIDName =
8852 Config.isTargetDevice()
8853 ? std::string(EntryFnName)
8854 : createPlatformSpecificName(Parts: {EntryFnName, "region_id"});
8855
8856 OutlinedFnID = registerTargetRegionFunction(EntryInfo, OutlinedFunction: OutlinedFn,
8857 EntryFnName, EntryFnIDName);
8858 return Error::success();
8859}
8860
8861Constant *OpenMPIRBuilder::registerTargetRegionFunction(
8862 TargetRegionEntryInfo &EntryInfo, Function *OutlinedFn,
8863 StringRef EntryFnName, StringRef EntryFnIDName) {
8864 if (OutlinedFn)
8865 setOutlinedTargetRegionFunctionAttributes(OutlinedFn);
8866 auto OutlinedFnID = createOutlinedFunctionID(OutlinedFn, EntryFnIDName);
8867 auto EntryAddr = createTargetRegionEntryAddr(OutlinedFn, EntryFnName);
8868 OffloadInfoManager.registerTargetRegionEntryInfo(
8869 EntryInfo, Addr: EntryAddr, ID: OutlinedFnID,
8870 Flags: OffloadEntriesInfoManager::OMPTargetRegionEntryTargetRegion);
8871 return OutlinedFnID;
8872}
8873
8874OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTargetData(
8875 const LocationDescription &Loc, InsertPointTy AllocaIP,
8876 InsertPointTy CodeGenIP, ArrayRef<BasicBlock *> DeallocBlocks,
8877 Value *DeviceID, Value *IfCond, TargetDataInfo &Info,
8878 GenMapInfoCallbackTy GenMapInfoCB, CustomMapperCallbackTy CustomMapperCB,
8879 omp::RuntimeFunction *MapperFunc,
8880 function_ref<InsertPointOrErrorTy(InsertPointTy CodeGenIP,
8881 BodyGenTy BodyGenType)>
8882 BodyGenCB,
8883 function_ref<void(unsigned int, Value *)> DeviceAddrCB, Value *SrcLocInfo) {
8884 if (!updateToLocation(Loc))
8885 return InsertPointTy();
8886
8887 Builder.restoreIP(IP: CodeGenIP);
8888
8889 bool IsStandAlone = !BodyGenCB;
8890 MapInfosTy *MapInfo;
8891 // Generate the code for the opening of the data environment. Capture all the
8892 // arguments of the runtime call by reference because they are used in the
8893 // closing of the region.
8894 auto BeginThenGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
8895 ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
8896 MapInfo = &GenMapInfoCB(Builder.saveIP());
8897 if (Error Err = emitOffloadingArrays(
8898 AllocaIP, CodeGenIP: Builder.saveIP(), CombinedInfo&: *MapInfo, Info, CustomMapperCB,
8899 /*IsNonContiguous=*/true, DeviceAddrCB))
8900 return Err;
8901
8902 TargetDataRTArgs RTArgs;
8903 emitOffloadingArraysArgument(Builder, RTArgs, Info);
8904
8905 // Emit the number of elements in the offloading arrays.
8906 Value *PointerNum = Builder.getInt32(C: Info.NumberOfPtrs);
8907
8908 // Source location for the ident struct
8909 if (!SrcLocInfo) {
8910 uint32_t SrcLocStrSize;
8911 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8912 SrcLocInfo = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8913 }
8914
8915 SmallVector<llvm::Value *, 13> OffloadingArgs = {
8916 SrcLocInfo, DeviceID,
8917 PointerNum, RTArgs.BasePointersArray,
8918 RTArgs.PointersArray, RTArgs.SizesArray,
8919 RTArgs.MapTypesArray, RTArgs.MapNamesArray,
8920 RTArgs.MappersArray};
8921
8922 if (IsStandAlone) {
8923 assert(MapperFunc && "MapperFunc missing for standalone target data");
8924
8925 auto TaskBodyCB = [&](Value *, Value *,
8926 IRBuilderBase::InsertPoint) -> Error {
8927 if (Info.HasNoWait) {
8928 OffloadingArgs.append(IL: {llvm::Constant::getNullValue(Ty: Int32),
8929 llvm::Constant::getNullValue(Ty: VoidPtr),
8930 llvm::Constant::getNullValue(Ty: Int32),
8931 llvm::Constant::getNullValue(Ty: VoidPtr)});
8932 }
8933
8934 createRuntimeFunctionCall(Callee: getOrCreateRuntimeFunctionPtr(FnID: *MapperFunc),
8935 Args: OffloadingArgs);
8936
8937 if (Info.HasNoWait) {
8938 BasicBlock *OffloadContBlock =
8939 BasicBlock::Create(Context&: Builder.getContext(), Name: "omp_offload.cont");
8940 Function *CurFn = Builder.GetInsertBlock()->getParent();
8941 emitBlock(BB: OffloadContBlock, CurFn, /*IsFinished=*/true);
8942 Builder.restoreIP(IP: Builder.saveIP());
8943 }
8944 return Error::success();
8945 };
8946
8947 bool RequiresOuterTargetTask = Info.HasNoWait;
8948 if (!RequiresOuterTargetTask)
8949 cantFail(Err: TaskBodyCB(/*DeviceID=*/nullptr, /*RTLoc=*/nullptr,
8950 /*TargetTaskAllocaIP=*/{}));
8951 else
8952 cantFail(ValOrErr: emitTargetTask(TaskBodyCB, DeviceID, RTLoc: SrcLocInfo, AllocaIP,
8953 /*Dependencies=*/{}, RTArgs, HasNoWait: Info.HasNoWait));
8954 } else {
8955 Function *BeginMapperFunc = getOrCreateRuntimeFunctionPtr(
8956 FnID: omp::OMPRTL___tgt_target_data_begin_mapper);
8957
8958 createRuntimeFunctionCall(Callee: BeginMapperFunc, Args: OffloadingArgs);
8959
8960 for (auto DeviceMap : Info.DevicePtrInfoMap) {
8961 if (isa<AllocaInst>(Val: DeviceMap.second.second)) {
8962 auto *LI =
8963 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: DeviceMap.second.first);
8964 Builder.CreateStore(Val: LI, Ptr: DeviceMap.second.second);
8965 }
8966 }
8967
8968 // If device pointer privatization is required, emit the body of the
8969 // region here. It will have to be duplicated: with and without
8970 // privatization.
8971 InsertPointOrErrorTy AfterIP =
8972 BodyGenCB(Builder.saveIP(), BodyGenTy::Priv);
8973 if (!AfterIP)
8974 return AfterIP.takeError();
8975 Builder.restoreIP(IP: *AfterIP);
8976 }
8977 return Error::success();
8978 };
8979
8980 // If we need device pointer privatization, we need to emit the body of the
8981 // region with no privatization in the 'else' branch of the conditional.
8982 // Otherwise, we don't have to do anything.
8983 auto BeginElseGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
8984 ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
8985 InsertPointOrErrorTy AfterIP =
8986 BodyGenCB(Builder.saveIP(), BodyGenTy::DupNoPriv);
8987 if (!AfterIP)
8988 return AfterIP.takeError();
8989 Builder.restoreIP(IP: *AfterIP);
8990 return Error::success();
8991 };
8992
8993 // Generate code for the closing of the data region.
8994 auto EndThenGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
8995 ArrayRef<BasicBlock *> DeallocBlocks) {
8996 TargetDataRTArgs RTArgs;
8997 Info.EmitDebug = !MapInfo->Names.empty();
8998 emitOffloadingArraysArgument(Builder, RTArgs, Info, /*ForEndCall=*/true);
8999
9000 // Emit the number of elements in the offloading arrays.
9001 Value *PointerNum = Builder.getInt32(C: Info.NumberOfPtrs);
9002
9003 // Source location for the ident struct
9004 if (!SrcLocInfo) {
9005 uint32_t SrcLocStrSize;
9006 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
9007 SrcLocInfo = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
9008 }
9009
9010 Value *OffloadingArgs[] = {SrcLocInfo, DeviceID,
9011 PointerNum, RTArgs.BasePointersArray,
9012 RTArgs.PointersArray, RTArgs.SizesArray,
9013 RTArgs.MapTypesArray, RTArgs.MapNamesArray,
9014 RTArgs.MappersArray};
9015 Function *EndMapperFunc =
9016 getOrCreateRuntimeFunctionPtr(FnID: omp::OMPRTL___tgt_target_data_end_mapper);
9017
9018 createRuntimeFunctionCall(Callee: EndMapperFunc, Args: OffloadingArgs);
9019 return Error::success();
9020 };
9021
9022 // We don't have to do anything to close the region if the if clause evaluates
9023 // to false.
9024 auto EndElseGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
9025 ArrayRef<BasicBlock *> DeallocBlocks) {
9026 return Error::success();
9027 };
9028
9029 Error Err = [&]() -> Error {
9030 if (BodyGenCB) {
9031 Error Err = [&]() {
9032 if (IfCond)
9033 return emitIfClause(Cond: IfCond, ThenGen: BeginThenGen, ElseGen: BeginElseGen, AllocaIP);
9034 return BeginThenGen(AllocaIP, Builder.saveIP(), DeallocBlocks);
9035 }();
9036
9037 if (Err)
9038 return Err;
9039
9040 // If we don't require privatization of device pointers, we emit the body
9041 // in between the runtime calls. This avoids duplicating the body code.
9042 InsertPointOrErrorTy AfterIP =
9043 BodyGenCB(Builder.saveIP(), BodyGenTy::NoPriv);
9044 if (!AfterIP)
9045 return AfterIP.takeError();
9046 restoreIPandDebugLoc(Builder, IP: *AfterIP);
9047
9048 if (IfCond)
9049 return emitIfClause(Cond: IfCond, ThenGen: EndThenGen, ElseGen: EndElseGen, AllocaIP);
9050 return EndThenGen(AllocaIP, Builder.saveIP(), DeallocBlocks);
9051 }
9052 if (IfCond)
9053 return emitIfClause(Cond: IfCond, ThenGen: BeginThenGen, ElseGen: EndElseGen, AllocaIP);
9054 return BeginThenGen(AllocaIP, Builder.saveIP(), DeallocBlocks);
9055 }();
9056
9057 if (Err)
9058 return Err;
9059
9060 return Builder.saveIP();
9061}
9062
9063FunctionCallee
9064OpenMPIRBuilder::createForStaticInitFunction(unsigned IVSize, bool IVSigned,
9065 bool IsGPUDistribute) {
9066 assert((IVSize == 32 || IVSize == 64) &&
9067 "IV size is not compatible with the omp runtime");
9068 RuntimeFunction Name;
9069 if (IsGPUDistribute)
9070 Name = IVSize == 32
9071 ? (IVSigned ? omp::OMPRTL___kmpc_distribute_static_init_4
9072 : omp::OMPRTL___kmpc_distribute_static_init_4u)
9073 : (IVSigned ? omp::OMPRTL___kmpc_distribute_static_init_8
9074 : omp::OMPRTL___kmpc_distribute_static_init_8u);
9075 else
9076 Name = IVSize == 32 ? (IVSigned ? omp::OMPRTL___kmpc_for_static_init_4
9077 : omp::OMPRTL___kmpc_for_static_init_4u)
9078 : (IVSigned ? omp::OMPRTL___kmpc_for_static_init_8
9079 : omp::OMPRTL___kmpc_for_static_init_8u);
9080
9081 return getOrCreateRuntimeFunction(M, FnID: Name);
9082}
9083
9084FunctionCallee OpenMPIRBuilder::createDispatchInitFunction(unsigned IVSize,
9085 bool IVSigned) {
9086 assert((IVSize == 32 || IVSize == 64) &&
9087 "IV size is not compatible with the omp runtime");
9088 RuntimeFunction Name = IVSize == 32
9089 ? (IVSigned ? omp::OMPRTL___kmpc_dispatch_init_4
9090 : omp::OMPRTL___kmpc_dispatch_init_4u)
9091 : (IVSigned ? omp::OMPRTL___kmpc_dispatch_init_8
9092 : omp::OMPRTL___kmpc_dispatch_init_8u);
9093
9094 return getOrCreateRuntimeFunction(M, FnID: Name);
9095}
9096
9097FunctionCallee OpenMPIRBuilder::createDispatchNextFunction(unsigned IVSize,
9098 bool IVSigned) {
9099 assert((IVSize == 32 || IVSize == 64) &&
9100 "IV size is not compatible with the omp runtime");
9101 RuntimeFunction Name = IVSize == 32
9102 ? (IVSigned ? omp::OMPRTL___kmpc_dispatch_next_4
9103 : omp::OMPRTL___kmpc_dispatch_next_4u)
9104 : (IVSigned ? omp::OMPRTL___kmpc_dispatch_next_8
9105 : omp::OMPRTL___kmpc_dispatch_next_8u);
9106
9107 return getOrCreateRuntimeFunction(M, FnID: Name);
9108}
9109
9110FunctionCallee OpenMPIRBuilder::createDispatchFiniFunction(unsigned IVSize,
9111 bool IVSigned) {
9112 assert((IVSize == 32 || IVSize == 64) &&
9113 "IV size is not compatible with the omp runtime");
9114 RuntimeFunction Name = IVSize == 32
9115 ? (IVSigned ? omp::OMPRTL___kmpc_dispatch_fini_4
9116 : omp::OMPRTL___kmpc_dispatch_fini_4u)
9117 : (IVSigned ? omp::OMPRTL___kmpc_dispatch_fini_8
9118 : omp::OMPRTL___kmpc_dispatch_fini_8u);
9119
9120 return getOrCreateRuntimeFunction(M, FnID: Name);
9121}
9122
9123FunctionCallee OpenMPIRBuilder::createDispatchDeinitFunction() {
9124 return getOrCreateRuntimeFunction(M, FnID: omp::OMPRTL___kmpc_dispatch_deinit);
9125}
9126
9127static void FixupDebugInfoForOutlinedFunction(
9128 OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, Function *Func,
9129 DenseMap<Value *, std::tuple<Value *, unsigned>> &ValueReplacementMap) {
9130
9131 DISubprogram *NewSP = Func->getSubprogram();
9132 if (!NewSP)
9133 return;
9134
9135 SmallDenseMap<DILocalVariable *, DILocalVariable *> RemappedVariables;
9136
9137 auto GetUpdatedDIVariable = [&](DILocalVariable *OldVar, unsigned arg) {
9138 DILocalVariable *&NewVar = RemappedVariables[OldVar];
9139 // Only use cached variable if the arg number matches. This is important
9140 // so that DIVariable created for privatized variables are not discarded.
9141 if (NewVar && (arg == NewVar->getArg()))
9142 return NewVar;
9143
9144 NewVar = llvm::DILocalVariable::get(
9145 Context&: Builder.getContext(), Scope: OldVar->getScope(), Name: OldVar->getName(),
9146 File: OldVar->getFile(), Line: OldVar->getLine(), Type: OldVar->getType(), Arg: arg,
9147 Flags: OldVar->getFlags(), AlignInBits: OldVar->getAlignInBits(), Annotations: OldVar->getAnnotations());
9148 return NewVar;
9149 };
9150
9151 auto UpdateDebugRecord = [&](auto *DR) {
9152 DILocalVariable *OldVar = DR->getVariable();
9153 unsigned ArgNo = 0;
9154 for (auto Loc : DR->location_ops()) {
9155 auto Iter = ValueReplacementMap.find(Loc);
9156 if (Iter != ValueReplacementMap.end()) {
9157 DR->replaceVariableLocationOp(Loc, std::get<0>(Iter->second));
9158 ArgNo = std::get<1>(Iter->second) + 1;
9159 }
9160 }
9161 if (ArgNo != 0)
9162 DR->setVariable(GetUpdatedDIVariable(OldVar, ArgNo));
9163 };
9164
9165 SmallVector<DbgVariableRecord *, 4> DVRsToDelete;
9166 auto MoveDebugRecordToCorrectBlock = [&](DbgVariableRecord *DVR) {
9167 if (DVR->getNumVariableLocationOps() != 1u) {
9168 DVR->setKillLocation();
9169 return;
9170 }
9171 Value *Loc = DVR->getVariableLocationOp(OpIdx: 0u);
9172 BasicBlock *CurBB = DVR->getParent();
9173 BasicBlock *RequiredBB = nullptr;
9174
9175 if (Instruction *LocInst = dyn_cast<Instruction>(Val: Loc))
9176 RequiredBB = LocInst->getParent();
9177 else if (isa<llvm::Argument>(Val: Loc))
9178 RequiredBB = &DVR->getFunction()->getEntryBlock();
9179
9180 if (RequiredBB && RequiredBB != CurBB) {
9181 assert(!RequiredBB->empty());
9182 RequiredBB->insertDbgRecordBefore(DR: DVR->clone(),
9183 Here: RequiredBB->back().getIterator());
9184 DVRsToDelete.push_back(Elt: DVR);
9185 }
9186 };
9187
9188 // The location and scope of variable intrinsics and records still point to
9189 // the parent function of the target region. Update them.
9190 for (Instruction &I : instructions(F: Func)) {
9191 assert(!isa<llvm::DbgVariableIntrinsic>(&I) &&
9192 "Unexpected debug intrinsic");
9193 for (DbgVariableRecord &DVR : filterDbgVars(R: I.getDbgRecordRange())) {
9194 UpdateDebugRecord(&DVR);
9195 MoveDebugRecordToCorrectBlock(&DVR);
9196 }
9197 }
9198 for (auto *DVR : DVRsToDelete)
9199 DVR->getMarker()->MarkedInstr->dropOneDbgRecord(I: DVR);
9200 // An extra argument is passed to the device. Create the debug data for it.
9201 if (OMPBuilder.Config.isTargetDevice()) {
9202 DICompileUnit *CU = NewSP->getUnit();
9203 Module *M = Func->getParent();
9204 DIBuilder DB(*M, true, CU);
9205 DIType *VoidPtrTy =
9206 DB.createQualifiedType(Tag: dwarf::DW_TAG_pointer_type, FromTy: nullptr);
9207 unsigned ArgNo = Func->arg_size();
9208 DILocalVariable *Var = DB.createParameterVariable(
9209 Scope: NewSP, Name: "dyn_ptr", ArgNo, File: NewSP->getFile(), /*LineNo=*/0, Ty: VoidPtrTy,
9210 /*AlwaysPreserve=*/false, Flags: DINode::DIFlags::FlagArtificial);
9211 auto Loc = DILocation::get(Context&: Func->getContext(), Line: 0, Column: 0, Scope: NewSP, InlinedAt: 0);
9212 Argument *LastArg = Func->getArg(i: Func->arg_size() - 1);
9213 DB.insertDeclare(Storage: LastArg, VarInfo: Var, Expr: DB.createExpression(), DL: Loc,
9214 InsertAtEnd: &(*Func->begin()));
9215 }
9216}
9217
9218static Value *removeASCastIfPresent(Value *V) {
9219 if (Operator::getOpcode(V) == Instruction::AddrSpaceCast)
9220 return cast<Operator>(Val: V)->getOperand(i: 0);
9221 return V;
9222}
9223
9224static Expected<Function *> createOutlinedFunction(
9225 OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder,
9226 const OpenMPIRBuilder::TargetKernelDefaultAttrs &DefaultAttrs,
9227 StringRef FuncName, SmallVectorImpl<Value *> &Inputs,
9228 OpenMPIRBuilder::TargetBodyGenCallbackTy &CBFunc,
9229 OpenMPIRBuilder::TargetGenArgAccessorsCallbackTy &ArgAccessorFuncCB) {
9230 SmallVector<Type *> ParameterTypes;
9231 if (OMPBuilder.Config.isTargetDevice()) {
9232 // All parameters to target devices are passed as pointers
9233 // or i64. This assumes 64-bit address spaces/pointers.
9234 for (auto &Arg : Inputs)
9235 ParameterTypes.push_back(Elt: Arg->getType()->isPointerTy()
9236 ? Arg->getType()
9237 : Type::getInt64Ty(C&: Builder.getContext()));
9238 } else {
9239 for (auto &Arg : Inputs)
9240 ParameterTypes.push_back(Elt: Arg->getType());
9241 }
9242
9243 // The implicit dyn_ptr argument is always the last parameter on both host
9244 // and device so the argument counts match without runtime manipulation.
9245 auto *PtrTy = PointerType::getUnqual(C&: Builder.getContext());
9246 ParameterTypes.push_back(Elt: PtrTy);
9247
9248 auto BB = Builder.GetInsertBlock();
9249 auto M = BB->getModule();
9250 auto FuncType = FunctionType::get(Result: Builder.getVoidTy(), Params: ParameterTypes,
9251 /*isVarArg*/ false);
9252 auto Func =
9253 Function::Create(Ty: FuncType, Linkage: GlobalValue::InternalLinkage, N: FuncName, M);
9254
9255 // Forward target-cpu and target-features function attributes from the
9256 // original function to the new outlined function.
9257 Function *ParentFn = Builder.GetInsertBlock()->getParent();
9258
9259 auto TargetCpuAttr = ParentFn->getFnAttribute(Kind: "target-cpu");
9260 if (TargetCpuAttr.isStringAttribute())
9261 Func->addFnAttr(Attr: TargetCpuAttr);
9262
9263 auto TargetFeaturesAttr = ParentFn->getFnAttribute(Kind: "target-features");
9264 if (TargetFeaturesAttr.isStringAttribute())
9265 Func->addFnAttr(Attr: TargetFeaturesAttr);
9266
9267 if (OMPBuilder.Config.isTargetDevice()) {
9268 Value *ExecMode =
9269 OMPBuilder.emitKernelExecutionMode(KernelName: FuncName, Mode: DefaultAttrs.ExecFlags);
9270 OMPBuilder.emitUsed(Name: "llvm.compiler.used", List: {ExecMode});
9271 }
9272
9273 // Save insert point.
9274 IRBuilder<>::InsertPointGuard IPG(Builder);
9275 // We will generate the entries in the outlined function but the debug
9276 // location may still be pointing to the parent function. Reset it now.
9277 Builder.SetCurrentDebugLocation(llvm::DebugLoc());
9278
9279 // Generate the region into the function.
9280 BasicBlock *EntryBB = BasicBlock::Create(Context&: Builder.getContext(), Name: "entry", Parent: Func);
9281 Builder.SetInsertPoint(EntryBB);
9282
9283 // Insert target init call in the device compilation pass.
9284 if (OMPBuilder.Config.isTargetDevice())
9285 Builder.restoreIP(IP: OMPBuilder.createTargetInit(Loc: Builder, Attrs: DefaultAttrs));
9286
9287 BasicBlock *UserCodeEntryBB = Builder.GetInsertBlock();
9288
9289 // As we embed the user code in the middle of our target region after we
9290 // generate entry code, we must move what allocas we can into the entry
9291 // block to avoid possible breaking optimisations for device
9292 if (OMPBuilder.Config.isTargetDevice())
9293 OMPBuilder.ConstantAllocaRaiseCandidates.emplace_back(Args&: Func);
9294
9295 BasicBlock *ExitBB = splitBB(Builder, /*CreateBranch=*/true, Name: "target.exit");
9296 BasicBlock *OutlinedBodyBB =
9297 splitBB(Builder, /*CreateBranch=*/true, Name: "outlined.body");
9298 llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = CBFunc(
9299 Builder.saveIP(),
9300 OpenMPIRBuilder::InsertPointTy(OutlinedBodyBB, OutlinedBodyBB->begin()),
9301 ExitBB);
9302 if (!AfterIP)
9303 return AfterIP.takeError();
9304 Builder.SetInsertPoint(ExitBB);
9305
9306 // Insert target deinit call in the device compilation pass.
9307 if (OMPBuilder.Config.isTargetDevice())
9308 OMPBuilder.createTargetDeinit(Loc: Builder);
9309
9310 // Insert return instruction.
9311 Builder.CreateRetVoid();
9312
9313 // New Alloca IP at entry point of created device function.
9314 Builder.SetInsertPoint(EntryBB->getFirstNonPHIIt());
9315 auto AllocaIP = Builder.saveIP();
9316
9317 Builder.SetInsertPoint(UserCodeEntryBB->getFirstNonPHIOrDbg());
9318
9319 // Do not include the artificial dyn_ptr argument.
9320 const auto &ArgRange = make_range(x: Func->arg_begin(), y: Func->arg_end() - 1);
9321
9322 DenseMap<Value *, std::tuple<Value *, unsigned>> ValueReplacementMap;
9323
9324 auto ReplaceValue = [](Value *Input, Value *InputCopy, Function *Func) {
9325 // Things like GEP's can come in the form of Constants. Constants and
9326 // ConstantExpr's do not have access to the knowledge of what they're
9327 // contained in, so we must dig a little to find an instruction so we
9328 // can tell if they're used inside of the function we're outlining. We
9329 // also replace the original constant expression with a new instruction
9330 // equivalent; an instruction as it allows easy modification in the
9331 // following loop, as we can now know the constant (instruction) is
9332 // owned by our target function and replaceUsesOfWith can now be invoked
9333 // on it (cannot do this with constants it seems). A brand new one also
9334 // allows us to be cautious as it is perhaps possible the old expression
9335 // was used inside of the function but exists and is used externally
9336 // (unlikely by the nature of a Constant, but still).
9337 // NOTE: We cannot remove dead constants that have been rewritten to
9338 // instructions at this stage, we run the risk of breaking later lowering
9339 // by doing so as we could still be in the process of lowering the module
9340 // from MLIR to LLVM-IR and the MLIR lowering may still require the original
9341 // constants we have created rewritten versions of.
9342 if (auto *Const = dyn_cast<Constant>(Val: Input))
9343 convertUsersOfConstantsToInstructions(Consts: Const, RestrictToFunc: Func, RemoveDeadConstants: false);
9344
9345 // Collect users before iterating over them to avoid invalidating the
9346 // iteration in case a user uses Input more than once (e.g. a call
9347 // instruction).
9348 SetVector<User *> Users(Input->users().begin(), Input->users().end());
9349 // Collect all the instructions
9350 for (User *User : make_early_inc_range(Range&: Users))
9351 if (auto *Instr = dyn_cast<Instruction>(Val: User))
9352 if (Instr->getFunction() == Func)
9353 Instr->replaceUsesOfWith(From: Input, To: InputCopy);
9354 };
9355
9356 SmallVector<std::pair<Value *, Value *>> DeferredReplacement;
9357
9358 // Rewrite uses of input valus to parameters.
9359 for (auto InArg : zip(t&: Inputs, u: ArgRange)) {
9360 Value *Input = std::get<0>(t&: InArg);
9361 Argument &Arg = std::get<1>(t&: InArg);
9362 Value *InputCopy = nullptr;
9363
9364 llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = ArgAccessorFuncCB(
9365 Arg, Input, InputCopy, AllocaIP, Builder.saveIP(),
9366 OpenMPIRBuilder::InsertPointTy(ExitBB, ExitBB->begin()));
9367 if (!AfterIP)
9368 return AfterIP.takeError();
9369 Builder.restoreIP(IP: *AfterIP);
9370 ValueReplacementMap[Input] = std::make_tuple(args&: InputCopy, args: Arg.getArgNo());
9371
9372 // In certain cases a Global may be set up for replacement, however, this
9373 // Global may be used in multiple arguments to the kernel, just segmented
9374 // apart, for example, if we have a global array, that is sectioned into
9375 // multiple mappings (technically not legal in OpenMP, but there is a case
9376 // in Fortran for Common Blocks where this is neccesary), we will end up
9377 // with GEP's into this array inside the kernel, that refer to the Global
9378 // but are technically separate arguments to the kernel for all intents and
9379 // purposes. If we have mapped a segment that requires a GEP into the 0-th
9380 // index, it will fold into an referal to the Global, if we then encounter
9381 // this folded GEP during replacement all of the references to the
9382 // Global in the kernel will be replaced with the argument we have generated
9383 // that corresponds to it, including any other GEP's that refer to the
9384 // Global that may be other arguments. This will invalidate all of the other
9385 // preceding mapped arguments that refer to the same global that may be
9386 // separate segments. To prevent this, we defer global processing until all
9387 // other processing has been performed.
9388 if (llvm::isa<llvm::GlobalValue, llvm::GlobalObject, llvm::GlobalVariable>(
9389 Val: removeASCastIfPresent(V: Input))) {
9390 DeferredReplacement.push_back(Elt: std::make_pair(x&: Input, y&: InputCopy));
9391 continue;
9392 }
9393
9394 if (isa<ConstantData>(Val: Input))
9395 continue;
9396
9397 ReplaceValue(Input, InputCopy, Func);
9398 }
9399
9400 // Replace all of our deferred Input values, currently just Globals.
9401 for (auto Deferred : DeferredReplacement)
9402 ReplaceValue(std::get<0>(in&: Deferred), std::get<1>(in&: Deferred), Func);
9403
9404 FixupDebugInfoForOutlinedFunction(OMPBuilder, Builder, Func,
9405 ValueReplacementMap);
9406 return Func;
9407}
9408/// Given a task descriptor, TaskWithPrivates, return the pointer to the block
9409/// of pointers containing shared data between the parent task and the created
9410/// task.
9411static LoadInst *loadSharedDataFromTaskDescriptor(OpenMPIRBuilder &OMPIRBuilder,
9412 IRBuilderBase &Builder,
9413 Value *TaskWithPrivates,
9414 Type *TaskWithPrivatesTy) {
9415
9416 Type *TaskTy = OMPIRBuilder.Task;
9417 LLVMContext &Ctx = Builder.getContext();
9418 Value *TaskT =
9419 Builder.CreateStructGEP(Ty: TaskWithPrivatesTy, Ptr: TaskWithPrivates, Idx: 0);
9420 Value *Shareds = TaskT;
9421 // TaskWithPrivatesTy can be one of the following
9422 // 1. %struct.task_with_privates = type { %struct.kmp_task_ompbuilder_t,
9423 // %struct.privates }
9424 // 2. %struct.kmp_task_ompbuilder_t ;; This is simply TaskTy
9425 //
9426 // In the former case, that is when TaskWithPrivatesTy != TaskTy,
9427 // its first member has to be the task descriptor. TaskTy is the type of the
9428 // task descriptor. TaskT is the pointer to the task descriptor. Loading the
9429 // first member of TaskT, gives us the pointer to shared data.
9430 if (TaskWithPrivatesTy != TaskTy)
9431 Shareds = Builder.CreateStructGEP(Ty: TaskTy, Ptr: TaskT, Idx: 0);
9432 return Builder.CreateLoad(Ty: PointerType::getUnqual(C&: Ctx), Ptr: Shareds);
9433}
9434/// Create an entry point for a target task with the following.
9435/// It'll have the following signature
9436/// void @.omp_target_task_proxy_func(i32 %thread.id, ptr %task)
9437/// This function is called from emitTargetTask once the
9438/// code to launch the target kernel has been outlined already.
9439/// NumOffloadingArrays is the number of offloading arrays that we need to copy
9440/// into the task structure so that the deferred target task can access this
9441/// data even after the stack frame of the generating task has been rolled
9442/// back. Offloading arrays contain base pointers, pointers, sizes etc
9443/// of the data that the target kernel will access. These in effect are the
9444/// non-empty arrays of pointers held by OpenMPIRBuilder::TargetDataRTArgs.
9445static Function *emitTargetTaskProxyFunction(
9446 OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, CallInst *StaleCI,
9447 StructType *PrivatesTy, StructType *TaskWithPrivatesTy,
9448 const size_t NumOffloadingArrays, const int SharedArgsOperandNo) {
9449
9450 // If NumOffloadingArrays is non-zero, PrivatesTy better not be nullptr.
9451 // This is because PrivatesTy is the type of the structure in which
9452 // we pass the offloading arrays to the deferred target task.
9453 assert((!NumOffloadingArrays || PrivatesTy) &&
9454 "PrivatesTy cannot be nullptr when there are offloadingArrays"
9455 "to privatize");
9456
9457 Module &M = OMPBuilder.M;
9458 // KernelLaunchFunction is the target launch function, i.e.
9459 // the function that sets up kernel arguments and calls
9460 // __tgt_target_kernel to launch the kernel on the device.
9461 //
9462 Function *KernelLaunchFunction = StaleCI->getCalledFunction();
9463
9464 // StaleCI is the CallInst which is the call to the outlined
9465 // target kernel launch function. If there are local live-in values
9466 // that the outlined function uses then these are aggregated into a structure
9467 // which is passed as the second argument. If there are no local live-in
9468 // values or if all values used by the outlined kernel are global variables,
9469 // then there's only one argument, the threadID. So, StaleCI can be
9470 //
9471 // %structArg = alloca { ptr, ptr }, align 8
9472 // %gep_ = getelementptr { ptr, ptr }, ptr %structArg, i32 0, i32 0
9473 // store ptr %20, ptr %gep_, align 8
9474 // %gep_8 = getelementptr { ptr, ptr }, ptr %structArg, i32 0, i32 1
9475 // store ptr %21, ptr %gep_8, align 8
9476 // call void @_QQmain..omp_par.1(i32 %global.tid.val6, ptr %structArg)
9477 //
9478 // OR
9479 //
9480 // call void @_QQmain..omp_par.1(i32 %global.tid.val6)
9481 OpenMPIRBuilder::InsertPointTy IP(StaleCI->getParent(),
9482 StaleCI->getIterator());
9483
9484 LLVMContext &Ctx = StaleCI->getParent()->getContext();
9485
9486 Type *ThreadIDTy = Type::getInt32Ty(C&: Ctx);
9487 Type *TaskPtrTy = OMPBuilder.TaskPtr;
9488 [[maybe_unused]] Type *TaskTy = OMPBuilder.Task;
9489
9490 auto ProxyFnTy =
9491 FunctionType::get(Result: Builder.getVoidTy(), Params: {ThreadIDTy, TaskPtrTy},
9492 /* isVarArg */ false);
9493 auto ProxyFn = Function::Create(Ty: ProxyFnTy, Linkage: GlobalValue::InternalLinkage,
9494 N: ".omp_target_task_proxy_func", M);
9495 Value *ThreadId = ProxyFn->getArg(i: 0);
9496 Value *TaskWithPrivates = ProxyFn->getArg(i: 1);
9497 ThreadId->setName("thread.id");
9498 TaskWithPrivates->setName("task");
9499
9500 bool HasShareds = SharedArgsOperandNo > 0;
9501 bool HasOffloadingArrays = NumOffloadingArrays > 0;
9502 IRBuilder<>::InsertPointGuard IPG(Builder);
9503 BasicBlock *EntryBB =
9504 BasicBlock::Create(Context&: Builder.getContext(), Name: "entry", Parent: ProxyFn);
9505 Builder.SetInsertPoint(EntryBB);
9506 Builder.SetCurrentDebugLocation(llvm::DebugLoc());
9507
9508 SmallVector<Value *> KernelLaunchArgs;
9509 KernelLaunchArgs.reserve(N: StaleCI->arg_size());
9510 KernelLaunchArgs.push_back(Elt: ThreadId);
9511
9512 if (HasOffloadingArrays) {
9513 assert(TaskTy != TaskWithPrivatesTy &&
9514 "If there are offloading arrays to pass to the target"
9515 "TaskTy cannot be the same as TaskWithPrivatesTy");
9516 (void)TaskTy;
9517 Value *Privates =
9518 Builder.CreateStructGEP(Ty: TaskWithPrivatesTy, Ptr: TaskWithPrivates, Idx: 1);
9519 for (unsigned int i = 0; i < NumOffloadingArrays; ++i)
9520 KernelLaunchArgs.push_back(
9521 Elt: Builder.CreateStructGEP(Ty: PrivatesTy, Ptr: Privates, Idx: i));
9522 }
9523
9524 if (HasShareds) {
9525 auto *ArgStructAlloca =
9526 dyn_cast<AllocaInst>(Val: StaleCI->getArgOperand(i: SharedArgsOperandNo));
9527 assert(ArgStructAlloca &&
9528 "Unable to find the alloca instruction corresponding to arguments "
9529 "for extracted function");
9530 auto *ArgStructType = cast<StructType>(Val: ArgStructAlloca->getAllocatedType());
9531 std::optional<TypeSize> ArgAllocSize =
9532 ArgStructAlloca->getAllocationSize(DL: M.getDataLayout());
9533 assert(ArgStructType && ArgAllocSize &&
9534 "Unable to determine size of arguments for extracted function");
9535 uint64_t StructSize = ArgAllocSize->getFixedValue();
9536
9537 AllocaInst *NewArgStructAlloca =
9538 Builder.CreateAlloca(Ty: ArgStructType, ArraySize: nullptr, Name: "structArg");
9539
9540 Value *SharedsSize = Builder.getInt64(C: StructSize);
9541
9542 LoadInst *LoadShared = loadSharedDataFromTaskDescriptor(
9543 OMPIRBuilder&: OMPBuilder, Builder, TaskWithPrivates, TaskWithPrivatesTy);
9544
9545 Builder.CreateMemCpy(
9546 Dst: NewArgStructAlloca, DstAlign: NewArgStructAlloca->getAlign(), Src: LoadShared,
9547 SrcAlign: LoadShared->getPointerAlignment(DL: M.getDataLayout()), Size: SharedsSize);
9548 KernelLaunchArgs.push_back(Elt: NewArgStructAlloca);
9549 }
9550 OMPBuilder.createRuntimeFunctionCall(Callee: KernelLaunchFunction, Args: KernelLaunchArgs);
9551 Builder.CreateRetVoid();
9552 return ProxyFn;
9553}
9554static Type *getOffloadingArrayType(Value *V) {
9555
9556 if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: V))
9557 return GEP->getSourceElementType();
9558 if (auto *Alloca = dyn_cast<AllocaInst>(Val: V))
9559 return Alloca->getAllocatedType();
9560
9561 llvm_unreachable("Unhandled Instruction type");
9562 return nullptr;
9563}
9564// This function returns a struct that has at most two members.
9565// The first member is always %struct.kmp_task_ompbuilder_t, that is the task
9566// descriptor. The second member, if needed, is a struct containing arrays
9567// that need to be passed to the offloaded target kernel. For example,
9568// if .offload_baseptrs, .offload_ptrs and .offload_sizes have to be passed to
9569// the target kernel and their types are [3 x ptr], [3 x ptr] and [3 x i64]
9570// respectively, then the types created by this function are
9571//
9572// %struct.privates = type { [3 x ptr], [3 x ptr], [3 x i64] }
9573// %struct.task_with_privates = type { %struct.kmp_task_ompbuilder_t,
9574// %struct.privates }
9575// %struct.task_with_privates is returned by this function.
9576// If there aren't any offloading arrays to pass to the target kernel,
9577// %struct.kmp_task_ompbuilder_t is returned.
9578static StructType *
9579createTaskWithPrivatesTy(OpenMPIRBuilder &OMPIRBuilder,
9580 ArrayRef<Value *> OffloadingArraysToPrivatize) {
9581
9582 if (OffloadingArraysToPrivatize.empty())
9583 return OMPIRBuilder.Task;
9584
9585 SmallVector<Type *, 4> StructFieldTypes;
9586 for (Value *V : OffloadingArraysToPrivatize) {
9587 assert(V->getType()->isPointerTy() &&
9588 "Expected pointer to array to privatize. Got a non-pointer value "
9589 "instead");
9590 Type *ArrayTy = getOffloadingArrayType(V);
9591 assert(ArrayTy && "ArrayType cannot be nullptr");
9592 StructFieldTypes.push_back(Elt: ArrayTy);
9593 }
9594 StructType *PrivatesStructTy =
9595 StructType::create(Elements: StructFieldTypes, Name: "struct.privates");
9596 return StructType::create(Elements: {OMPIRBuilder.Task, PrivatesStructTy},
9597 Name: "struct.task_with_privates");
9598}
9599static Error emitTargetOutlinedFunction(
9600 OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, bool IsOffloadEntry,
9601 TargetRegionEntryInfo &EntryInfo,
9602 const OpenMPIRBuilder::TargetKernelDefaultAttrs &DefaultAttrs,
9603 Function *&OutlinedFn, Constant *&OutlinedFnID,
9604 SmallVectorImpl<Value *> &Inputs,
9605 OpenMPIRBuilder::TargetBodyGenCallbackTy &CBFunc,
9606 OpenMPIRBuilder::TargetGenArgAccessorsCallbackTy &ArgAccessorFuncCB) {
9607
9608 OpenMPIRBuilder::FunctionGenCallback &&GenerateOutlinedFunction =
9609 [&](StringRef EntryFnName) {
9610 return createOutlinedFunction(OMPBuilder, Builder, DefaultAttrs,
9611 FuncName: EntryFnName, Inputs, CBFunc,
9612 ArgAccessorFuncCB);
9613 };
9614
9615 return OMPBuilder.emitTargetRegionFunction(
9616 EntryInfo, GenerateFunctionCallback&: GenerateOutlinedFunction, IsOffloadEntry, OutlinedFn,
9617 OutlinedFnID);
9618}
9619
9620OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitTargetTask(
9621 TargetTaskBodyCallbackTy TaskBodyCB, Value *DeviceID, Value *RTLoc,
9622 OpenMPIRBuilder::InsertPointTy AllocaIP,
9623 const DependenciesInfo &Dependencies, const TargetDataRTArgs &RTArgs,
9624 bool HasNoWait) {
9625
9626 // The following explains the code-gen scenario for the `target` directive. A
9627 // similar scneario is followed for other device-related directives (e.g.
9628 // `target enter data`) but in similar fashion since we only need to emit task
9629 // that encapsulates the proper runtime call.
9630 //
9631 // When we arrive at this function, the target region itself has been
9632 // outlined into the function OutlinedFn.
9633 // So at ths point, for
9634 // --------------------------------------------------------------
9635 // void user_code_that_offloads(...) {
9636 // omp target depend(..) map(from:a) map(to:b) private(i)
9637 // do i = 1, 10
9638 // a(i) = b(i) + n
9639 // }
9640 //
9641 // --------------------------------------------------------------
9642 //
9643 // we have
9644 //
9645 // --------------------------------------------------------------
9646 //
9647 // void user_code_that_offloads(...) {
9648 // %.offload_baseptrs = alloca [2 x ptr], align 8
9649 // %.offload_ptrs = alloca [2 x ptr], align 8
9650 // %.offload_mappers = alloca [2 x ptr], align 8
9651 // ;; target region has been outlined and now we need to
9652 // ;; offload to it via a target task.
9653 // }
9654 // void outlined_device_function(ptr a, ptr b, ptr n) {
9655 // n = *n_ptr;
9656 // do i = 1, 10
9657 // a(i) = b(i) + n
9658 // }
9659 //
9660 // We have to now do the following
9661 // (i) Make an offloading call to outlined_device_function using the OpenMP
9662 // RTL. See 'kernel_launch_function' in the pseudo code below. This is
9663 // emitted by emitKernelLaunch
9664 // (ii) Create a task entry point function that calls kernel_launch_function
9665 // and is the entry point for the target task. See
9666 // '@.omp_target_task_proxy_func in the pseudocode below.
9667 // (iii) Create a task with the task entry point created in (ii)
9668 //
9669 // That is we create the following
9670 // struct task_with_privates {
9671 // struct kmp_task_ompbuilder_t task_struct;
9672 // struct privates {
9673 // [2 x ptr] ; baseptrs
9674 // [2 x ptr] ; ptrs
9675 // [2 x i64] ; sizes
9676 // }
9677 // }
9678 // void user_code_that_offloads(...) {
9679 // %.offload_baseptrs = alloca [2 x ptr], align 8
9680 // %.offload_ptrs = alloca [2 x ptr], align 8
9681 // %.offload_sizes = alloca [2 x i64], align 8
9682 //
9683 // %structArg = alloca { ptr, ptr, ptr }, align 8
9684 // %strucArg[0] = a
9685 // %strucArg[1] = b
9686 // %strucArg[2] = &n
9687 //
9688 // target_task_with_privates = @__kmpc_omp_target_task_alloc(...,
9689 // sizeof(kmp_task_ompbuilder_t),
9690 // sizeof(structArg),
9691 // @.omp_target_task_proxy_func,
9692 // ...)
9693 // memcpy(target_task_with_privates->task_struct->shareds, %structArg,
9694 // sizeof(structArg))
9695 // memcpy(target_task_with_privates->privates->baseptrs,
9696 // offload_baseptrs, sizeof(offload_baseptrs)
9697 // memcpy(target_task_with_privates->privates->ptrs,
9698 // offload_ptrs, sizeof(offload_ptrs)
9699 // memcpy(target_task_with_privates->privates->sizes,
9700 // offload_sizes, sizeof(offload_sizes)
9701 // dependencies_array = ...
9702 // ;; if nowait not present
9703 // call @__kmpc_omp_wait_deps(..., dependencies_array)
9704 // call @__kmpc_omp_task_begin_if0(...)
9705 // call @ @.omp_target_task_proxy_func(i32 thread_id, ptr
9706 // %target_task_with_privates)
9707 // call @__kmpc_omp_task_complete_if0(...)
9708 // }
9709 //
9710 // define internal void @.omp_target_task_proxy_func(i32 %thread.id,
9711 // ptr %task) {
9712 // %structArg = alloca {ptr, ptr, ptr}
9713 // %task_ptr = getelementptr(%task, 0, 0)
9714 // %shared_data = load (getelementptr %task_ptr, 0, 0)
9715 // mempcy(%structArg, %shared_data, sizeof(%structArg))
9716 //
9717 // %offloading_arrays = getelementptr(%task, 0, 1)
9718 // %offload_baseptrs = getelementptr(%offloading_arrays, 0, 0)
9719 // %offload_ptrs = getelementptr(%offloading_arrays, 0, 1)
9720 // %offload_sizes = getelementptr(%offloading_arrays, 0, 2)
9721 // kernel_launch_function(%thread.id, %offload_baseptrs, %offload_ptrs,
9722 // %offload_sizes, %structArg)
9723 // }
9724 //
9725 // We need the proxy function because the signature of the task entry point
9726 // expected by kmpc_omp_task is always the same and will be different from
9727 // that of the kernel_launch function.
9728 //
9729 // kernel_launch_function is generated by emitKernelLaunch and has the
9730 // always_inline attribute. For this example, it'll look like so:
9731 // void kernel_launch_function(%thread_id, %offload_baseptrs, %offload_ptrs,
9732 // %offload_sizes, %structArg) alwaysinline {
9733 // %kernel_args = alloca %struct.__tgt_kernel_arguments, align 8
9734 // ; load aggregated data from %structArg
9735 // ; setup kernel_args using offload_baseptrs, offload_ptrs and
9736 // ; offload_sizes
9737 // call i32 @__tgt_target_kernel(...,
9738 // outlined_device_function,
9739 // ptr %kernel_args)
9740 // }
9741 // void outlined_device_function(ptr a, ptr b, ptr n) {
9742 // n = *n_ptr;
9743 // do i = 1, 10
9744 // a(i) = b(i) + n
9745 // }
9746 //
9747 BasicBlock *TargetTaskBodyBB =
9748 splitBB(Builder, /*CreateBranch=*/true, Name: "target.task.body");
9749 BasicBlock *TargetTaskAllocaBB =
9750 splitBB(Builder, /*CreateBranch=*/true, Name: "target.task.alloca");
9751
9752 InsertPointTy TargetTaskAllocaIP(TargetTaskAllocaBB,
9753 TargetTaskAllocaBB->begin());
9754 InsertPointTy TargetTaskBodyIP(TargetTaskBodyBB, TargetTaskBodyBB->begin());
9755
9756 auto OI = std::make_unique<OutlineInfo>();
9757 OI->EntryBB = TargetTaskAllocaBB;
9758 OI->OuterAllocBB = AllocaIP.getBlock();
9759
9760 // Add the thread ID argument.
9761 SmallVector<Instruction *, 4> ToBeDeleted;
9762 OI->ExcludeArgsFromAggregate.push_back(Elt: createFakeIntVal(
9763 Builder, OuterAllocaIP: AllocaIP, ToBeDeleted, InnerAllocaIP: TargetTaskAllocaIP, Name: "global.tid", AsPtr: false));
9764
9765 // Generate the task body which will subsequently be outlined.
9766 Builder.restoreIP(IP: TargetTaskBodyIP);
9767 if (Error Err = TaskBodyCB(DeviceID, RTLoc, TargetTaskAllocaIP))
9768 return Err;
9769
9770 // The outliner (CodeExtractor) extract a sequence or vector of blocks that
9771 // it is given. These blocks are enumerated by
9772 // OpenMPIRBuilder::OutlineInfo::collectBlocks which expects the OI.ExitBlock
9773 // to be outside the region. In other words, OI.ExitBlock is expected to be
9774 // the start of the region after the outlining. We used to set OI.ExitBlock
9775 // to the InsertBlock after TaskBodyCB is done. This is fine in most cases
9776 // except when the task body is a single basic block. In that case,
9777 // OI.ExitBlock is set to the single task body block and will get left out of
9778 // the outlining process. So, simply create a new empty block to which we
9779 // uncoditionally branch from where TaskBodyCB left off
9780 OI->ExitBB = BasicBlock::Create(Context&: Builder.getContext(), Name: "target.task.cont");
9781 emitBlock(BB: OI->ExitBB, CurFn: Builder.GetInsertBlock()->getParent(),
9782 /*IsFinished=*/true);
9783
9784 SmallVector<Value *, 2> OffloadingArraysToPrivatize;
9785 bool NeedsTargetTask = HasNoWait && DeviceID;
9786 if (NeedsTargetTask) {
9787 for (auto *V :
9788 {RTArgs.BasePointersArray, RTArgs.PointersArray, RTArgs.MappersArray,
9789 RTArgs.MapNamesArray, RTArgs.MapTypesArray, RTArgs.MapTypesArrayEnd,
9790 RTArgs.SizesArray}) {
9791 if (V && !isa<ConstantPointerNull, GlobalVariable>(Val: V)) {
9792 OffloadingArraysToPrivatize.push_back(Elt: V);
9793 OI->ExcludeArgsFromAggregate.push_back(Elt: V);
9794 }
9795 }
9796 }
9797 OI->PostOutlineCB = [this, ToBeDeleted, Dependencies, NeedsTargetTask,
9798 DeviceID, OffloadingArraysToPrivatize](
9799 Function &OutlinedFn) mutable {
9800 assert(OutlinedFn.hasOneUse() &&
9801 "there must be a single user for the outlined function");
9802
9803 CallInst *StaleCI = cast<CallInst>(Val: OutlinedFn.user_back());
9804
9805 // The first argument of StaleCI is always the thread id.
9806 // The next few arguments are the pointers to offloading arrays
9807 // if any. (see OffloadingArraysToPrivatize)
9808 // Finally, all other local values that are live-in into the outlined region
9809 // end up in a structure whose pointer is passed as the last argument. This
9810 // piece of data is passed in the "shared" field of the task structure. So,
9811 // we know we have to pass shareds to the task if the number of arguments is
9812 // greater than OffloadingArraysToPrivatize.size() + 1 The 1 is for the
9813 // thread id. Further, for safety, we assert that the number of arguments of
9814 // StaleCI is exactly OffloadingArraysToPrivatize.size() + 2
9815 const unsigned int NumStaleCIArgs = StaleCI->arg_size();
9816 bool HasShareds = NumStaleCIArgs > OffloadingArraysToPrivatize.size() + 1;
9817 assert((!HasShareds ||
9818 NumStaleCIArgs == (OffloadingArraysToPrivatize.size() + 2)) &&
9819 "Wrong number of arguments for StaleCI when shareds are present");
9820 int SharedArgOperandNo =
9821 HasShareds ? OffloadingArraysToPrivatize.size() + 1 : 0;
9822
9823 StructType *TaskWithPrivatesTy =
9824 createTaskWithPrivatesTy(OMPIRBuilder&: *this, OffloadingArraysToPrivatize);
9825 StructType *PrivatesTy = nullptr;
9826
9827 if (!OffloadingArraysToPrivatize.empty())
9828 PrivatesTy =
9829 static_cast<StructType *>(TaskWithPrivatesTy->getElementType(N: 1));
9830
9831 Function *ProxyFn = emitTargetTaskProxyFunction(
9832 OMPBuilder&: *this, Builder, StaleCI, PrivatesTy, TaskWithPrivatesTy,
9833 NumOffloadingArrays: OffloadingArraysToPrivatize.size(), SharedArgsOperandNo: SharedArgOperandNo);
9834
9835 LLVM_DEBUG(dbgs() << "Proxy task entry function created: " << *ProxyFn
9836 << "\n");
9837
9838 Builder.SetInsertPoint(StaleCI);
9839
9840 // Gather the arguments for emitting the runtime call.
9841 uint32_t SrcLocStrSize;
9842 Constant *SrcLocStr =
9843 getOrCreateSrcLocStr(Loc: LocationDescription(Builder), SrcLocStrSize);
9844 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
9845
9846 // @__kmpc_omp_task_alloc or @__kmpc_omp_target_task_alloc
9847 //
9848 // If `HasNoWait == true`, we call @__kmpc_omp_target_task_alloc to provide
9849 // the DeviceID to the deferred task and also since
9850 // @__kmpc_omp_target_task_alloc creates an untied/async task.
9851 Function *TaskAllocFn =
9852 !NeedsTargetTask
9853 ? getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_alloc)
9854 : getOrCreateRuntimeFunctionPtr(
9855 FnID: OMPRTL___kmpc_omp_target_task_alloc);
9856
9857 // Arguments - `loc_ref` (Ident) and `gtid` (ThreadID)
9858 // call.
9859 Value *ThreadID = getOrCreateThreadID(Ident);
9860
9861 // Argument - `sizeof_kmp_task_t` (TaskSize)
9862 // Tasksize refers to the size in bytes of kmp_task_t data structure
9863 // plus any other data to be passed to the target task, if any, which
9864 // is packed into a struct. kmp_task_t and the struct so created are
9865 // packed into a wrapper struct whose type is TaskWithPrivatesTy.
9866 Value *TaskSize = Builder.getInt64(
9867 C: M.getDataLayout().getTypeStoreSize(Ty: TaskWithPrivatesTy));
9868
9869 // Argument - `sizeof_shareds` (SharedsSize)
9870 // SharedsSize refers to the shareds array size in the kmp_task_t data
9871 // structure.
9872 Value *SharedsSize = Builder.getInt64(C: 0);
9873 if (HasShareds) {
9874 auto *ArgStructAlloca =
9875 dyn_cast<AllocaInst>(Val: StaleCI->getArgOperand(i: SharedArgOperandNo));
9876 assert(ArgStructAlloca &&
9877 "Unable to find the alloca instruction corresponding to arguments "
9878 "for extracted function");
9879 std::optional<TypeSize> ArgAllocSize =
9880 ArgStructAlloca->getAllocationSize(DL: M.getDataLayout());
9881 assert(ArgAllocSize &&
9882 "Unable to determine size of arguments for extracted function");
9883 SharedsSize = Builder.getInt64(C: ArgAllocSize->getFixedValue());
9884 }
9885
9886 // Argument - `flags`
9887 // Task is tied iff (Flags & 1) == 1.
9888 // Task is untied iff (Flags & 1) == 0.
9889 // Task is final iff (Flags & 2) == 2.
9890 // Task is not final iff (Flags & 2) == 0.
9891 // A target task is not final and is untied.
9892 Value *Flags = Builder.getInt32(C: 0);
9893
9894 // Emit the @__kmpc_omp_task_alloc runtime call
9895 // The runtime call returns a pointer to an area where the task captured
9896 // variables must be copied before the task is run (TaskData)
9897 CallInst *TaskData = nullptr;
9898
9899 SmallVector<llvm::Value *> TaskAllocArgs = {
9900 /*loc_ref=*/Ident, /*gtid=*/ThreadID,
9901 /*flags=*/Flags,
9902 /*sizeof_task=*/TaskSize, /*sizeof_shared=*/SharedsSize,
9903 /*task_func=*/ProxyFn};
9904
9905 if (NeedsTargetTask) {
9906 assert(DeviceID && "Expected non-empty device ID.");
9907 TaskAllocArgs.push_back(Elt: DeviceID);
9908 }
9909
9910 TaskData = createRuntimeFunctionCall(Callee: TaskAllocFn, Args: TaskAllocArgs);
9911
9912 Align Alignment = TaskData->getPointerAlignment(DL: M.getDataLayout());
9913 if (HasShareds) {
9914 Value *Shareds = StaleCI->getArgOperand(i: SharedArgOperandNo);
9915 Value *TaskShareds = loadSharedDataFromTaskDescriptor(
9916 OMPIRBuilder&: *this, Builder, TaskWithPrivates: TaskData, TaskWithPrivatesTy);
9917 Builder.CreateMemCpy(Dst: TaskShareds, DstAlign: Alignment, Src: Shareds, SrcAlign: Alignment,
9918 Size: SharedsSize);
9919 }
9920 if (!OffloadingArraysToPrivatize.empty()) {
9921 Value *Privates =
9922 Builder.CreateStructGEP(Ty: TaskWithPrivatesTy, Ptr: TaskData, Idx: 1);
9923 for (unsigned int i = 0; i < OffloadingArraysToPrivatize.size(); ++i) {
9924 Value *PtrToPrivatize = OffloadingArraysToPrivatize[i];
9925 [[maybe_unused]] Type *ArrayType =
9926 getOffloadingArrayType(V: PtrToPrivatize);
9927 assert(ArrayType && "ArrayType cannot be nullptr");
9928
9929 Type *ElementType = PrivatesTy->getElementType(N: i);
9930 assert(ElementType == ArrayType &&
9931 "ElementType should match ArrayType");
9932 (void)ArrayType;
9933
9934 Value *Dst = Builder.CreateStructGEP(Ty: PrivatesTy, Ptr: Privates, Idx: i);
9935 Builder.CreateMemCpy(
9936 Dst, DstAlign: Alignment, Src: PtrToPrivatize, SrcAlign: Alignment,
9937 Size: Builder.getInt64(C: M.getDataLayout().getTypeStoreSize(Ty: ElementType)));
9938 }
9939 }
9940
9941 Value *DepArray = nullptr;
9942 Value *NumDeps = nullptr;
9943 if (Dependencies.DepArray) {
9944 DepArray = Dependencies.DepArray;
9945 NumDeps = Dependencies.NumDeps;
9946 } else if (!Dependencies.Deps.empty()) {
9947 DepArray = emitTaskDependencies(OMPBuilder&: *this, Dependencies: Dependencies.Deps);
9948 NumDeps = Builder.getInt32(C: Dependencies.Deps.size());
9949 }
9950
9951 // ---------------------------------------------------------------
9952 // V5.2 13.8 target construct
9953 // If the nowait clause is present, execution of the target task
9954 // may be deferred. If the nowait clause is not present, the target task is
9955 // an included task.
9956 // ---------------------------------------------------------------
9957 // The above means that the lack of a nowait on the target construct
9958 // translates to '#pragma omp task if(0)'
9959 if (!NeedsTargetTask) {
9960 if (DepArray) {
9961 Function *TaskWaitFn =
9962 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_wait_deps);
9963 createRuntimeFunctionCall(
9964 Callee: TaskWaitFn,
9965 Args: {/*loc_ref=*/Ident, /*gtid=*/ThreadID,
9966 /*ndeps=*/NumDeps,
9967 /*dep_list=*/DepArray,
9968 /*ndeps_noalias=*/ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0),
9969 /*noalias_dep_list=*/
9970 ConstantPointerNull::get(T: PointerType::getUnqual(C&: M.getContext()))});
9971 }
9972 // Included task.
9973 Function *TaskBeginFn =
9974 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_begin_if0);
9975 Function *TaskCompleteFn =
9976 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_complete_if0);
9977 createRuntimeFunctionCall(Callee: TaskBeginFn, Args: {Ident, ThreadID, TaskData});
9978 CallInst *CI = createRuntimeFunctionCall(Callee: ProxyFn, Args: {ThreadID, TaskData});
9979 CI->setDebugLoc(StaleCI->getDebugLoc());
9980 createRuntimeFunctionCall(Callee: TaskCompleteFn, Args: {Ident, ThreadID, TaskData});
9981 } else if (DepArray) {
9982 // HasNoWait - meaning the task may be deferred. Call
9983 // __kmpc_omp_task_with_deps if there are dependencies,
9984 // else call __kmpc_omp_task
9985 Function *TaskFn =
9986 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_with_deps);
9987 createRuntimeFunctionCall(
9988 Callee: TaskFn,
9989 Args: {Ident, ThreadID, TaskData, NumDeps, DepArray,
9990 ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0),
9991 ConstantPointerNull::get(T: PointerType::getUnqual(C&: M.getContext()))});
9992 } else {
9993 // Emit the @__kmpc_omp_task runtime call to spawn the task
9994 Function *TaskFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task);
9995 createRuntimeFunctionCall(Callee: TaskFn, Args: {Ident, ThreadID, TaskData});
9996 }
9997
9998 Builder.ClearInsertionPoint();
9999 StaleCI->eraseFromParent();
10000 for (Instruction *I : llvm::reverse(C&: ToBeDeleted))
10001 I->eraseFromParent();
10002 };
10003 addOutlineInfo(OI: std::move(OI));
10004
10005 LLVM_DEBUG(dbgs() << "Insert block after emitKernelLaunch = \n"
10006 << *(Builder.GetInsertBlock()) << "\n");
10007 LLVM_DEBUG(dbgs() << "Module after emitKernelLaunch = \n"
10008 << *(Builder.GetInsertBlock()->getParent()->getParent())
10009 << "\n");
10010 return Builder.saveIP();
10011}
10012
10013Error OpenMPIRBuilder::emitOffloadingArraysAndArgs(
10014 InsertPointTy AllocaIP, InsertPointTy CodeGenIP, TargetDataInfo &Info,
10015 TargetDataRTArgs &RTArgs, MapInfosTy &CombinedInfo,
10016 CustomMapperCallbackTy CustomMapperCB, bool IsNonContiguous,
10017 bool ForEndCall, function_ref<void(unsigned int, Value *)> DeviceAddrCB) {
10018 if (Error Err =
10019 emitOffloadingArrays(AllocaIP, CodeGenIP, CombinedInfo, Info,
10020 CustomMapperCB, IsNonContiguous, DeviceAddrCB))
10021 return Err;
10022 emitOffloadingArraysArgument(Builder, RTArgs, Info, ForEndCall);
10023 return Error::success();
10024}
10025
10026static void emitTargetCall(
10027 OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder,
10028 OpenMPIRBuilder::InsertPointTy AllocaIP,
10029 ArrayRef<BasicBlock *> DeallocBlocks, OpenMPIRBuilder::TargetDataInfo &Info,
10030 const OpenMPIRBuilder::TargetKernelDefaultAttrs &DefaultAttrs,
10031 const OpenMPIRBuilder::TargetKernelRuntimeAttrs &RuntimeAttrs,
10032 Value *IfCond, Function *OutlinedFn, Constant *OutlinedFnID,
10033 SmallVectorImpl<Value *> &Args,
10034 OpenMPIRBuilder::GenMapInfoCallbackTy GenMapInfoCB,
10035 OpenMPIRBuilder::CustomMapperCallbackTy CustomMapperCB,
10036 const OpenMPIRBuilder::DependenciesInfo &Dependencies, bool HasNoWait,
10037 Value *DynCGroupMem, OMPDynGroupprivateFallbackType DynCGroupMemFallback) {
10038 // Generate a function call to the host fallback implementation of the target
10039 // region. This is called by the host when no offload entry was generated for
10040 // the target region and when the offloading call fails at runtime.
10041 auto &&EmitTargetCallFallbackCB = [&](OpenMPIRBuilder::InsertPointTy IP)
10042 -> OpenMPIRBuilder::InsertPointOrErrorTy {
10043 Builder.restoreIP(IP);
10044 // Ensure the host fallback has the same dyn_ptr ABI as the device.
10045 SmallVector<Value *> FallbackArgs(Args.begin(), Args.end());
10046 FallbackArgs.push_back(
10047 Elt: Constant::getNullValue(Ty: PointerType::getUnqual(C&: Builder.getContext())));
10048 OMPBuilder.createRuntimeFunctionCall(Callee: OutlinedFn, Args: FallbackArgs);
10049 return Builder.saveIP();
10050 };
10051
10052 bool HasDependencies = !Dependencies.empty();
10053 bool RequiresOuterTargetTask = HasNoWait || HasDependencies;
10054
10055 OpenMPIRBuilder::TargetKernelArgs KArgs;
10056
10057 auto TaskBodyCB =
10058 [&](Value *DeviceID, Value *RTLoc,
10059 IRBuilderBase::InsertPoint TargetTaskAllocaIP) -> Error {
10060 // Assume no error was returned because EmitTargetCallFallbackCB doesn't
10061 // produce any.
10062 llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(ValOrErr: [&]() {
10063 // emitKernelLaunch makes the necessary runtime call to offload the
10064 // kernel. We then outline all that code into a separate function
10065 // ('kernel_launch_function' in the pseudo code above). This function is
10066 // then called by the target task proxy function (see
10067 // '@.omp_target_task_proxy_func' in the pseudo code above)
10068 // "@.omp_target_task_proxy_func' is generated by
10069 // emitTargetTaskProxyFunction.
10070 if (OutlinedFnID && DeviceID)
10071 return OMPBuilder.emitKernelLaunch(Loc: Builder, OutlinedFnID,
10072 EmitTargetCallFallbackCB, Args&: KArgs,
10073 DeviceID, RTLoc, AllocaIP: TargetTaskAllocaIP);
10074
10075 // We only need to do the outlining if `DeviceID` is set to avoid calling
10076 // `emitKernelLaunch` if we want to code-gen for the host; e.g. if we are
10077 // generating the `else` branch of an `if` clause.
10078 //
10079 // When OutlinedFnID is set to nullptr, then it's not an offloading call.
10080 // In this case, we execute the host implementation directly.
10081 return EmitTargetCallFallbackCB(OMPBuilder.Builder.saveIP());
10082 }());
10083
10084 OMPBuilder.Builder.restoreIP(IP: AfterIP);
10085 return Error::success();
10086 };
10087
10088 auto &&EmitTargetCallElse =
10089 [&](OpenMPIRBuilder::InsertPointTy AllocaIP,
10090 OpenMPIRBuilder::InsertPointTy CodeGenIP,
10091 ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
10092 // Assume no error was returned because EmitTargetCallFallbackCB doesn't
10093 // produce any.
10094 OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(ValOrErr: [&]() {
10095 if (RequiresOuterTargetTask) {
10096 // Arguments that are intended to be directly forwarded to an
10097 // emitKernelLaunch call are pased as nullptr, since
10098 // OutlinedFnID=nullptr results in that call not being done.
10099 OpenMPIRBuilder::TargetDataRTArgs EmptyRTArgs;
10100 return OMPBuilder.emitTargetTask(TaskBodyCB, /*DeviceID=*/nullptr,
10101 /*RTLoc=*/nullptr, AllocaIP,
10102 Dependencies, RTArgs: EmptyRTArgs, HasNoWait);
10103 }
10104 return EmitTargetCallFallbackCB(Builder.saveIP());
10105 }());
10106
10107 Builder.restoreIP(IP: AfterIP);
10108 return Error::success();
10109 };
10110
10111 auto &&EmitTargetCallThen =
10112 [&](OpenMPIRBuilder::InsertPointTy AllocaIP,
10113 OpenMPIRBuilder::InsertPointTy CodeGenIP,
10114 ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
10115 Info.HasNoWait = HasNoWait;
10116 OpenMPIRBuilder::MapInfosTy &MapInfo = GenMapInfoCB(Builder.saveIP());
10117
10118 OpenMPIRBuilder::TargetDataRTArgs RTArgs;
10119 if (Error Err = OMPBuilder.emitOffloadingArraysAndArgs(
10120 AllocaIP, CodeGenIP: Builder.saveIP(), Info, RTArgs, CombinedInfo&: MapInfo, CustomMapperCB,
10121 /*IsNonContiguous=*/true,
10122 /*ForEndCall=*/false))
10123 return Err;
10124
10125 SmallVector<Value *, 3> NumTeamsC;
10126 for (auto [DefaultVal, RuntimeVal] :
10127 zip_equal(t: DefaultAttrs.MaxTeams, u: RuntimeAttrs.MaxTeams))
10128 NumTeamsC.push_back(Elt: RuntimeVal ? RuntimeVal
10129 : Builder.getInt32(C: DefaultVal));
10130
10131 // Calculate number of threads: 0 if no clauses specified, otherwise it is
10132 // the minimum between optional THREAD_LIMIT and NUM_THREADS clauses.
10133 auto InitMaxThreadsClause = [&Builder](Value *Clause) {
10134 if (Clause)
10135 Clause = Builder.CreateIntCast(V: Clause, DestTy: Builder.getInt32Ty(),
10136 /*isSigned=*/false);
10137 return Clause;
10138 };
10139 auto CombineMaxThreadsClauses = [&Builder](Value *Clause, Value *&Result) {
10140 if (Clause)
10141 Result =
10142 Result ? Builder.CreateSelect(C: Builder.CreateICmpULT(LHS: Result, RHS: Clause),
10143 True: Result, False: Clause)
10144 : Clause;
10145 };
10146
10147 // If a multi-dimensional THREAD_LIMIT is set, it is the OMPX_BARE case, so
10148 // the NUM_THREADS clause is overriden by THREAD_LIMIT.
10149 SmallVector<Value *, 3> NumThreadsC;
10150 Value *MaxThreadsClause =
10151 RuntimeAttrs.TeamsThreadLimit.size() == 1
10152 ? InitMaxThreadsClause(RuntimeAttrs.MaxThreads.front())
10153 : nullptr;
10154
10155 for (auto [TeamsVal, TargetVal] : zip_equal(
10156 t: RuntimeAttrs.TeamsThreadLimit, u: RuntimeAttrs.TargetThreadLimit)) {
10157 Value *TeamsThreadLimitClause = InitMaxThreadsClause(TeamsVal);
10158 Value *NumThreads = InitMaxThreadsClause(TargetVal);
10159
10160 CombineMaxThreadsClauses(TeamsThreadLimitClause, NumThreads);
10161 CombineMaxThreadsClauses(MaxThreadsClause, NumThreads);
10162
10163 NumThreadsC.push_back(Elt: NumThreads ? NumThreads : Builder.getInt32(C: 0));
10164 }
10165
10166 unsigned NumTargetItems = Info.NumberOfPtrs;
10167 uint32_t SrcLocStrSize;
10168 Constant *SrcLocStr = OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
10169 Value *RTLoc = OMPBuilder.getOrCreateIdent(SrcLocStr, SrcLocStrSize,
10170 LocFlags: llvm::omp::IdentFlag(0), Reserve2Flags: 0);
10171
10172 Value *TripCount = RuntimeAttrs.LoopTripCount
10173 ? Builder.CreateIntCast(V: RuntimeAttrs.LoopTripCount,
10174 DestTy: Builder.getInt64Ty(),
10175 /*isSigned=*/false)
10176 : Builder.getInt64(C: 0);
10177
10178 // Request zero groupprivate bytes by default.
10179 if (!DynCGroupMem)
10180 DynCGroupMem = Builder.getInt32(C: 0);
10181
10182 KArgs = OpenMPIRBuilder::TargetKernelArgs(
10183 NumTargetItems, RTArgs, TripCount, NumTeamsC, NumThreadsC, DynCGroupMem,
10184 HasNoWait, /*StrictBlocks=*/false, /*StrictThreads=*/false,
10185 DynCGroupMemFallback);
10186
10187 // Assume no error was returned because TaskBodyCB and
10188 // EmitTargetCallFallbackCB don't produce any.
10189 OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(ValOrErr: [&]() {
10190 // The presence of certain clauses on the target directive require the
10191 // explicit generation of the target task.
10192 if (RequiresOuterTargetTask)
10193 return OMPBuilder.emitTargetTask(TaskBodyCB, DeviceID: RuntimeAttrs.DeviceID,
10194 RTLoc, AllocaIP, Dependencies,
10195 RTArgs: KArgs.RTArgs, HasNoWait: Info.HasNoWait);
10196
10197 return OMPBuilder.emitKernelLaunch(
10198 Loc: Builder, OutlinedFnID, EmitTargetCallFallbackCB, Args&: KArgs,
10199 DeviceID: RuntimeAttrs.DeviceID, RTLoc, AllocaIP);
10200 }());
10201
10202 Builder.restoreIP(IP: AfterIP);
10203 return Error::success();
10204 };
10205
10206 // If we don't have an ID for the target region, it means an offload entry
10207 // wasn't created. In this case we just run the host fallback directly and
10208 // ignore any potential 'if' clauses.
10209 if (!OutlinedFnID) {
10210 cantFail(Err: EmitTargetCallElse(AllocaIP, Builder.saveIP(), DeallocBlocks));
10211 return;
10212 }
10213
10214 // If there's no 'if' clause, only generate the kernel launch code path.
10215 if (!IfCond) {
10216 cantFail(Err: EmitTargetCallThen(AllocaIP, Builder.saveIP(), DeallocBlocks));
10217 return;
10218 }
10219
10220 cantFail(Err: OMPBuilder.emitIfClause(Cond: IfCond, ThenGen: EmitTargetCallThen,
10221 ElseGen: EmitTargetCallElse, AllocaIP));
10222}
10223
10224OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTarget(
10225 const LocationDescription &Loc, bool IsOffloadEntry, InsertPointTy AllocaIP,
10226 InsertPointTy CodeGenIP, ArrayRef<BasicBlock *> DeallocBlocks,
10227 TargetDataInfo &Info, TargetRegionEntryInfo &EntryInfo,
10228 const TargetKernelDefaultAttrs &DefaultAttrs,
10229 const TargetKernelRuntimeAttrs &RuntimeAttrs, Value *IfCond,
10230 SmallVectorImpl<Value *> &Inputs, GenMapInfoCallbackTy GenMapInfoCB,
10231 OpenMPIRBuilder::TargetBodyGenCallbackTy CBFunc,
10232 OpenMPIRBuilder::TargetGenArgAccessorsCallbackTy ArgAccessorFuncCB,
10233 CustomMapperCallbackTy CustomMapperCB, const DependenciesInfo &Dependencies,
10234 bool HasNowait, Value *DynCGroupMem,
10235 OMPDynGroupprivateFallbackType DynCGroupMemFallback) {
10236
10237 if (!updateToLocation(Loc))
10238 return InsertPointTy();
10239
10240 Builder.restoreIP(IP: CodeGenIP);
10241
10242 Function *OutlinedFn;
10243 Constant *OutlinedFnID = nullptr;
10244 // The target region is outlined into its own function. The LLVM IR for
10245 // the target region itself is generated using the callbacks CBFunc
10246 // and ArgAccessorFuncCB
10247 if (Error Err = emitTargetOutlinedFunction(
10248 OMPBuilder&: *this, Builder, IsOffloadEntry, EntryInfo, DefaultAttrs, OutlinedFn,
10249 OutlinedFnID, Inputs, CBFunc, ArgAccessorFuncCB))
10250 return Err;
10251
10252 // If we are not on the target device, then we need to generate code
10253 // to make a remote call (offload) to the previously outlined function
10254 // that represents the target region. Do that now.
10255 if (!Config.isTargetDevice())
10256 emitTargetCall(OMPBuilder&: *this, Builder, AllocaIP, DeallocBlocks, Info, DefaultAttrs,
10257 RuntimeAttrs, IfCond, OutlinedFn, OutlinedFnID, Args&: Inputs,
10258 GenMapInfoCB, CustomMapperCB, Dependencies, HasNoWait: HasNowait,
10259 DynCGroupMem, DynCGroupMemFallback);
10260 return Builder.saveIP();
10261}
10262
10263std::string OpenMPIRBuilder::getNameWithSeparators(ArrayRef<StringRef> Parts,
10264 StringRef FirstSeparator,
10265 StringRef Separator) {
10266 SmallString<128> Buffer;
10267 llvm::raw_svector_ostream OS(Buffer);
10268 StringRef Sep = FirstSeparator;
10269 for (StringRef Part : Parts) {
10270 OS << Sep << Part;
10271 Sep = Separator;
10272 }
10273 return OS.str().str();
10274}
10275
10276std::string
10277OpenMPIRBuilder::createPlatformSpecificName(ArrayRef<StringRef> Parts) const {
10278 return OpenMPIRBuilder::getNameWithSeparators(Parts, FirstSeparator: Config.firstSeparator(),
10279 Separator: Config.separator());
10280}
10281
10282GlobalVariable *OpenMPIRBuilder::getOrCreateInternalVariable(
10283 Type *Ty, const StringRef &Name, std::optional<unsigned> AddressSpace) {
10284 auto &Elem = *InternalVars.try_emplace(Key: Name, Args: nullptr).first;
10285 if (Elem.second) {
10286 assert(Elem.second->getValueType() == Ty &&
10287 "OMP internal variable has different type than requested");
10288 } else {
10289 // TODO: investigate the appropriate linkage type used for the global
10290 // variable for possibly changing that to internal or private, or maybe
10291 // create different versions of the function for different OMP internal
10292 // variables.
10293 const DataLayout &DL = M.getDataLayout();
10294 // TODO: Investigate why AMDGPU expects AS 0 for globals even though the
10295 // default global AS is 1.
10296 // See double-target-call-with-declare-target.f90 and
10297 // declare-target-vars-in-target-region.f90 libomptarget
10298 // tests.
10299 unsigned AddressSpaceVal = AddressSpace ? *AddressSpace
10300 : M.getTargetTriple().isAMDGPU()
10301 ? 0
10302 : DL.getDefaultGlobalsAddressSpace();
10303 auto Linkage = this->M.getTargetTriple().isWasm()
10304 ? GlobalValue::InternalLinkage
10305 : GlobalValue::CommonLinkage;
10306 auto *GV = new GlobalVariable(M, Ty, /*IsConstant=*/false, Linkage,
10307 Constant::getNullValue(Ty), Elem.first(),
10308 /*InsertBefore=*/nullptr,
10309 GlobalValue::NotThreadLocal, AddressSpaceVal);
10310 const llvm::Align TypeAlign = DL.getABITypeAlign(Ty);
10311 const llvm::Align PtrAlign = DL.getPointerABIAlignment(AS: AddressSpaceVal);
10312 GV->setAlignment(std::max(a: TypeAlign, b: PtrAlign));
10313 Elem.second = GV;
10314 }
10315
10316 return Elem.second;
10317}
10318
10319Value *OpenMPIRBuilder::getOMPCriticalRegionLock(StringRef CriticalName) {
10320 std::string Prefix = Twine("gomp_critical_user_", CriticalName).str();
10321 std::string Name = getNameWithSeparators(Parts: {Prefix, "var"}, FirstSeparator: ".", Separator: ".");
10322 return getOrCreateInternalVariable(Ty: KmpCriticalNameTy, Name);
10323}
10324
10325Value *OpenMPIRBuilder::getSizeInBytes(Value *BasePtr) {
10326 LLVMContext &Ctx = Builder.getContext();
10327 Value *Null =
10328 Constant::getNullValue(Ty: PointerType::getUnqual(C&: BasePtr->getContext()));
10329 Value *SizeGep =
10330 Builder.CreateGEP(Ty: BasePtr->getType(), Ptr: Null, IdxList: Builder.getInt32(C: 1));
10331 Value *SizePtrToInt = Builder.CreatePtrToInt(V: SizeGep, DestTy: Type::getInt64Ty(C&: Ctx));
10332 return SizePtrToInt;
10333}
10334
10335GlobalVariable *
10336OpenMPIRBuilder::createOffloadMaptypes(SmallVectorImpl<uint64_t> &Mappings,
10337 std::string VarName) {
10338 llvm::Constant *MaptypesArrayInit =
10339 llvm::ConstantDataArray::get(Context&: M.getContext(), Elts&: Mappings);
10340 auto *MaptypesArrayGlobal = new llvm::GlobalVariable(
10341 M, MaptypesArrayInit->getType(),
10342 /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage, MaptypesArrayInit,
10343 VarName);
10344 MaptypesArrayGlobal->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
10345 return MaptypesArrayGlobal;
10346}
10347
10348void OpenMPIRBuilder::createMapperAllocas(const LocationDescription &Loc,
10349 InsertPointTy AllocaIP,
10350 unsigned NumOperands,
10351 struct MapperAllocas &MapperAllocas) {
10352 if (!updateToLocation(Loc))
10353 return;
10354
10355 auto *ArrI8PtrTy = ArrayType::get(ElementType: Int8Ptr, NumElements: NumOperands);
10356 auto *ArrI64Ty = ArrayType::get(ElementType: Int64, NumElements: NumOperands);
10357 Builder.restoreIP(IP: AllocaIP);
10358 AllocaInst *ArgsBase = Builder.CreateAlloca(
10359 Ty: ArrI8PtrTy, /* ArraySize = */ nullptr, Name: ".offload_baseptrs");
10360 AllocaInst *Args = Builder.CreateAlloca(Ty: ArrI8PtrTy, /* ArraySize = */ nullptr,
10361 Name: ".offload_ptrs");
10362 AllocaInst *ArgSizes = Builder.CreateAlloca(
10363 Ty: ArrI64Ty, /* ArraySize = */ nullptr, Name: ".offload_sizes");
10364 updateToLocation(Loc);
10365 MapperAllocas.ArgsBase = ArgsBase;
10366 MapperAllocas.Args = Args;
10367 MapperAllocas.ArgSizes = ArgSizes;
10368}
10369
10370void OpenMPIRBuilder::emitMapperCall(const LocationDescription &Loc,
10371 Function *MapperFunc, Value *SrcLocInfo,
10372 Value *MaptypesArg, Value *MapnamesArg,
10373 struct MapperAllocas &MapperAllocas,
10374 int64_t DeviceID, unsigned NumOperands) {
10375 if (!updateToLocation(Loc))
10376 return;
10377
10378 auto *ArrI8PtrTy = ArrayType::get(ElementType: Int8Ptr, NumElements: NumOperands);
10379 auto *ArrI64Ty = ArrayType::get(ElementType: Int64, NumElements: NumOperands);
10380 Value *ArgsBaseGEP =
10381 Builder.CreateInBoundsGEP(Ty: ArrI8PtrTy, Ptr: MapperAllocas.ArgsBase,
10382 IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: 0)});
10383 Value *ArgsGEP =
10384 Builder.CreateInBoundsGEP(Ty: ArrI8PtrTy, Ptr: MapperAllocas.Args,
10385 IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: 0)});
10386 Value *ArgSizesGEP =
10387 Builder.CreateInBoundsGEP(Ty: ArrI64Ty, Ptr: MapperAllocas.ArgSizes,
10388 IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: 0)});
10389 Value *NullPtr =
10390 Constant::getNullValue(Ty: PointerType::getUnqual(C&: Int8Ptr->getContext()));
10391 createRuntimeFunctionCall(Callee: MapperFunc, Args: {SrcLocInfo, Builder.getInt64(C: DeviceID),
10392 Builder.getInt32(C: NumOperands),
10393 ArgsBaseGEP, ArgsGEP, ArgSizesGEP,
10394 MaptypesArg, MapnamesArg, NullPtr});
10395}
10396
10397void OpenMPIRBuilder::emitOffloadingArraysArgument(IRBuilderBase &Builder,
10398 TargetDataRTArgs &RTArgs,
10399 TargetDataInfo &Info,
10400 bool ForEndCall) {
10401 assert((!ForEndCall || Info.separateBeginEndCalls()) &&
10402 "expected region end call to runtime only when end call is separate");
10403 auto UnqualPtrTy = PointerType::getUnqual(C&: M.getContext());
10404 auto VoidPtrTy = UnqualPtrTy;
10405 auto VoidPtrPtrTy = UnqualPtrTy;
10406 auto Int64Ty = Type::getInt64Ty(C&: M.getContext());
10407 auto Int64PtrTy = UnqualPtrTy;
10408
10409 if (!Info.NumberOfPtrs) {
10410 RTArgs.BasePointersArray = ConstantPointerNull::get(T: VoidPtrPtrTy);
10411 RTArgs.PointersArray = ConstantPointerNull::get(T: VoidPtrPtrTy);
10412 RTArgs.SizesArray = ConstantPointerNull::get(T: Int64PtrTy);
10413 RTArgs.MapTypesArray = ConstantPointerNull::get(T: Int64PtrTy);
10414 RTArgs.MapNamesArray = ConstantPointerNull::get(T: VoidPtrPtrTy);
10415 RTArgs.MappersArray = ConstantPointerNull::get(T: VoidPtrPtrTy);
10416 return;
10417 }
10418
10419 RTArgs.BasePointersArray = Builder.CreateConstInBoundsGEP2_32(
10420 Ty: ArrayType::get(ElementType: VoidPtrTy, NumElements: Info.NumberOfPtrs),
10421 Ptr: Info.RTArgs.BasePointersArray,
10422 /*Idx0=*/0, /*Idx1=*/0);
10423 RTArgs.PointersArray = Builder.CreateConstInBoundsGEP2_32(
10424 Ty: ArrayType::get(ElementType: VoidPtrTy, NumElements: Info.NumberOfPtrs), Ptr: Info.RTArgs.PointersArray,
10425 /*Idx0=*/0,
10426 /*Idx1=*/0);
10427 RTArgs.SizesArray = Builder.CreateConstInBoundsGEP2_32(
10428 Ty: ArrayType::get(ElementType: Int64Ty, NumElements: Info.NumberOfPtrs), Ptr: Info.RTArgs.SizesArray,
10429 /*Idx0=*/0, /*Idx1=*/0);
10430 RTArgs.MapTypesArray = Builder.CreateConstInBoundsGEP2_32(
10431 Ty: ArrayType::get(ElementType: Int64Ty, NumElements: Info.NumberOfPtrs),
10432 Ptr: ForEndCall && Info.RTArgs.MapTypesArrayEnd ? Info.RTArgs.MapTypesArrayEnd
10433 : Info.RTArgs.MapTypesArray,
10434 /*Idx0=*/0,
10435 /*Idx1=*/0);
10436
10437 // Only emit the mapper information arrays if debug information is
10438 // requested.
10439 if (!Info.EmitDebug)
10440 RTArgs.MapNamesArray = ConstantPointerNull::get(T: VoidPtrPtrTy);
10441 else
10442 RTArgs.MapNamesArray = Builder.CreateConstInBoundsGEP2_32(
10443 Ty: ArrayType::get(ElementType: VoidPtrTy, NumElements: Info.NumberOfPtrs), Ptr: Info.RTArgs.MapNamesArray,
10444 /*Idx0=*/0,
10445 /*Idx1=*/0);
10446 // If there is no user-defined mapper, set the mapper array to nullptr to
10447 // avoid an unnecessary data privatization
10448 if (!Info.HasMapper)
10449 RTArgs.MappersArray = ConstantPointerNull::get(T: VoidPtrPtrTy);
10450 else
10451 RTArgs.MappersArray =
10452 Builder.CreatePointerCast(V: Info.RTArgs.MappersArray, DestTy: VoidPtrPtrTy);
10453}
10454
10455void OpenMPIRBuilder::emitNonContiguousDescriptor(InsertPointTy AllocaIP,
10456 InsertPointTy CodeGenIP,
10457 MapInfosTy &CombinedInfo,
10458 TargetDataInfo &Info) {
10459 MapInfosTy::StructNonContiguousInfo &NonContigInfo =
10460 CombinedInfo.NonContigInfo;
10461
10462 // Build an array of struct descriptor_dim and then assign it to
10463 // offload_args.
10464 //
10465 // struct descriptor_dim {
10466 // uint64_t offset;
10467 // uint64_t count;
10468 // uint64_t stride
10469 // };
10470 Type *Int64Ty = Builder.getInt64Ty();
10471 StructType *DimTy = StructType::create(
10472 Context&: M.getContext(), Elements: ArrayRef<Type *>({Int64Ty, Int64Ty, Int64Ty}),
10473 Name: "struct.descriptor_dim");
10474
10475 enum { OffsetFD = 0, CountFD, StrideFD };
10476 // We need two index variable here since the size of "Dims" is the same as
10477 // the size of Components, however, the size of offset, count, and stride is
10478 // equal to the size of base declaration that is non-contiguous.
10479 for (unsigned I = 0, L = 0, E = NonContigInfo.Dims.size(); I < E; ++I) {
10480 // Skip emitting ir if dimension size is 1 since it cannot be
10481 // non-contiguous.
10482 if (NonContigInfo.Dims[I] == 1)
10483 continue;
10484 Builder.restoreIP(IP: AllocaIP);
10485 ArrayType *ArrayTy = ArrayType::get(ElementType: DimTy, NumElements: NonContigInfo.Dims[I]);
10486 AllocaInst *DimsAddr =
10487 Builder.CreateAlloca(Ty: ArrayTy, /* ArraySize = */ nullptr, Name: "dims");
10488 Builder.restoreIP(IP: CodeGenIP);
10489 for (unsigned II = 0, EE = NonContigInfo.Dims[I]; II < EE; ++II) {
10490 unsigned RevIdx = EE - II - 1;
10491 Value *DimsLVal = Builder.CreateInBoundsGEP(
10492 Ty: ArrayTy, Ptr: DimsAddr, IdxList: {Builder.getInt64(C: 0), Builder.getInt64(C: II)});
10493 // Offset
10494 Value *OffsetLVal = Builder.CreateStructGEP(Ty: DimTy, Ptr: DimsLVal, Idx: OffsetFD);
10495 Builder.CreateAlignedStore(
10496 Val: NonContigInfo.Offsets[L][RevIdx], Ptr: OffsetLVal,
10497 Align: M.getDataLayout().getPrefTypeAlign(Ty: OffsetLVal->getType()));
10498 // Count
10499 Value *CountLVal = Builder.CreateStructGEP(Ty: DimTy, Ptr: DimsLVal, Idx: CountFD);
10500 Builder.CreateAlignedStore(
10501 Val: NonContigInfo.Counts[L][RevIdx], Ptr: CountLVal,
10502 Align: M.getDataLayout().getPrefTypeAlign(Ty: CountLVal->getType()));
10503 // Stride
10504 Value *StrideLVal = Builder.CreateStructGEP(Ty: DimTy, Ptr: DimsLVal, Idx: StrideFD);
10505 Builder.CreateAlignedStore(
10506 Val: NonContigInfo.Strides[L][RevIdx], Ptr: StrideLVal,
10507 Align: M.getDataLayout().getPrefTypeAlign(Ty: CountLVal->getType()));
10508 }
10509 // args[I] = &dims
10510 Builder.restoreIP(IP: CodeGenIP);
10511 Value *DAddr = Builder.CreatePointerBitCastOrAddrSpaceCast(
10512 V: DimsAddr, DestTy: Builder.getPtrTy());
10513 Value *P = Builder.CreateConstInBoundsGEP2_32(
10514 Ty: ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: Info.NumberOfPtrs),
10515 Ptr: Info.RTArgs.PointersArray, Idx0: 0, Idx1: I);
10516 Builder.CreateAlignedStore(
10517 Val: DAddr, Ptr: P, Align: M.getDataLayout().getPrefTypeAlign(Ty: Builder.getPtrTy()));
10518 ++L;
10519 }
10520}
10521
10522void OpenMPIRBuilder::emitUDMapperArrayInitOrDel(
10523 Function *MapperFn, Value *MapperHandle, Value *Base, Value *Begin,
10524 Value *Size, Value *MapType, Value *MapName, TypeSize ElementSize,
10525 BasicBlock *ExitBB, bool IsInit) {
10526 StringRef Prefix = IsInit ? ".init" : ".del";
10527
10528 // Evaluate if this is an array section.
10529 BasicBlock *BodyBB = BasicBlock::Create(
10530 Context&: M.getContext(), Name: createPlatformSpecificName(Parts: {"omp.array", Prefix}));
10531 Value *IsArray =
10532 Builder.CreateICmpSGT(LHS: Size, RHS: Builder.getInt64(C: 1), Name: "omp.arrayinit.isarray");
10533 Value *DeleteBit = Builder.CreateAnd(
10534 LHS: MapType,
10535 RHS: Builder.getInt64(
10536 C: static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10537 OpenMPOffloadMappingFlags::OMP_MAP_DELETE)));
10538 Value *DeleteCond;
10539 Value *Cond;
10540 if (IsInit) {
10541 // base != begin?
10542 Value *BaseIsBegin = Builder.CreateICmpNE(LHS: Base, RHS: Begin);
10543 Cond = Builder.CreateOr(LHS: IsArray, RHS: BaseIsBegin);
10544 DeleteCond = Builder.CreateIsNull(
10545 Arg: DeleteBit,
10546 Name: createPlatformSpecificName(Parts: {"omp.array", Prefix, ".delete"}));
10547 } else {
10548 Cond = IsArray;
10549 DeleteCond = Builder.CreateIsNotNull(
10550 Arg: DeleteBit,
10551 Name: createPlatformSpecificName(Parts: {"omp.array", Prefix, ".delete"}));
10552 }
10553 Cond = Builder.CreateAnd(LHS: Cond, RHS: DeleteCond);
10554 Builder.CreateCondBr(Cond, True: BodyBB, False: ExitBB);
10555
10556 emitBlock(BB: BodyBB, CurFn: MapperFn);
10557 // Get the array size by multiplying element size and element number (i.e., \p
10558 // Size).
10559 Value *ArraySize = Builder.CreateNUWMul(LHS: Size, RHS: Builder.getInt64(C: ElementSize));
10560 // Remove OMP_MAP_TO and OMP_MAP_FROM from the map type, so that it achieves
10561 // memory allocation/deletion purpose only.
10562 Value *MapTypeArg = Builder.CreateAnd(
10563 LHS: MapType,
10564 RHS: Builder.getInt64(
10565 C: ~static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10566 OpenMPOffloadMappingFlags::OMP_MAP_TO |
10567 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
10568 MapTypeArg = Builder.CreateOr(
10569 LHS: MapTypeArg,
10570 RHS: Builder.getInt64(
10571 C: static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10572 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT)));
10573
10574 // Call the runtime API __tgt_push_mapper_component to fill up the runtime
10575 // data structure.
10576 Value *OffloadingArgs[] = {MapperHandle, Base, Begin,
10577 ArraySize, MapTypeArg, MapName};
10578 createRuntimeFunctionCall(
10579 Callee: getOrCreateRuntimeFunction(M, FnID: OMPRTL___tgt_push_mapper_component),
10580 Args: OffloadingArgs);
10581}
10582
10583Expected<Function *> OpenMPIRBuilder::emitUserDefinedMapper(
10584 function_ref<MapInfosOrErrorTy(InsertPointTy CodeGenIP, llvm::Value *PtrPHI,
10585 llvm::Value *BeginArg)>
10586 GenMapInfoCB,
10587 Type *ElemTy, StringRef FuncName, CustomMapperCallbackTy CustomMapperCB,
10588 bool PreserveMemberOfFlags, bool PropagatePresentToPointee) {
10589 SmallVector<Type *> Params;
10590 Params.emplace_back(Args: Builder.getPtrTy());
10591 Params.emplace_back(Args: Builder.getPtrTy());
10592 Params.emplace_back(Args: Builder.getPtrTy());
10593 Params.emplace_back(Args: Builder.getInt64Ty());
10594 Params.emplace_back(Args: Builder.getInt64Ty());
10595 Params.emplace_back(Args: Builder.getPtrTy());
10596
10597 auto *FnTy =
10598 FunctionType::get(Result: Builder.getVoidTy(), Params, /* IsVarArg */ isVarArg: false);
10599
10600 SmallString<64> TyStr;
10601 raw_svector_ostream Out(TyStr);
10602 Function *MapperFn =
10603 Function::Create(Ty: FnTy, Linkage: GlobalValue::InternalLinkage, N: FuncName, M);
10604 MapperFn->addFnAttr(Kind: Attribute::NoInline);
10605 MapperFn->addFnAttr(Kind: Attribute::NoUnwind);
10606 MapperFn->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
10607 MapperFn->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
10608 MapperFn->addParamAttr(ArgNo: 2, Kind: Attribute::NoUndef);
10609 MapperFn->addParamAttr(ArgNo: 3, Kind: Attribute::NoUndef);
10610 MapperFn->addParamAttr(ArgNo: 4, Kind: Attribute::NoUndef);
10611 MapperFn->addParamAttr(ArgNo: 5, Kind: Attribute::NoUndef);
10612
10613 // Start the mapper function code generation.
10614 BasicBlock *EntryBB = BasicBlock::Create(Context&: M.getContext(), Name: "entry", Parent: MapperFn);
10615 IRBuilder<>::InsertPointGuard IPG(Builder);
10616 Builder.SetInsertPoint(EntryBB);
10617 Builder.SetCurrentDebugLocation(llvm::DebugLoc());
10618
10619 Value *MapperHandle = MapperFn->getArg(i: 0);
10620 Value *BaseIn = MapperFn->getArg(i: 1);
10621 Value *BeginIn = MapperFn->getArg(i: 2);
10622 Value *Size = MapperFn->getArg(i: 3);
10623 Value *MapType = MapperFn->getArg(i: 4);
10624 Value *MapName = MapperFn->getArg(i: 5);
10625
10626 // Compute the starting and end addresses of array elements.
10627 // Prepare common arguments for array initiation and deletion.
10628 // Convert the size in bytes into the number of array elements.
10629 TypeSize ElementSize = M.getDataLayout().getTypeStoreSize(Ty: ElemTy);
10630 Size = Builder.CreateExactUDiv(LHS: Size, RHS: Builder.getInt64(C: ElementSize));
10631 Value *PtrBegin = BeginIn;
10632 Value *PtrEnd = Builder.CreateGEP(Ty: ElemTy, Ptr: PtrBegin, IdxList: Size);
10633
10634 // Emit array initiation if this is an array section and \p MapType indicates
10635 // that memory allocation is required.
10636 BasicBlock *HeadBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp.arraymap.head");
10637 emitUDMapperArrayInitOrDel(MapperFn, MapperHandle, Base: BaseIn, Begin: BeginIn, Size,
10638 MapType, MapName, ElementSize, ExitBB: HeadBB,
10639 /*IsInit=*/true);
10640
10641 // Emit a for loop to iterate through SizeArg of elements and map all of them.
10642
10643 // Emit the loop header block.
10644 emitBlock(BB: HeadBB, CurFn: MapperFn);
10645 BasicBlock *BodyBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp.arraymap.body");
10646 BasicBlock *DoneBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp.done");
10647 // Evaluate whether the initial condition is satisfied.
10648 Value *IsEmpty =
10649 Builder.CreateICmpEQ(LHS: PtrBegin, RHS: PtrEnd, Name: "omp.arraymap.isempty");
10650 Builder.CreateCondBr(Cond: IsEmpty, True: DoneBB, False: BodyBB);
10651
10652 // Emit the loop body block.
10653 emitBlock(BB: BodyBB, CurFn: MapperFn);
10654 BasicBlock *LastBB = BodyBB;
10655 PHINode *PtrPHI =
10656 Builder.CreatePHI(Ty: PtrBegin->getType(), NumReservedValues: 2, Name: "omp.arraymap.ptrcurrent");
10657 PtrPHI->addIncoming(V: PtrBegin, BB: HeadBB);
10658
10659 // Get map clause information. Fill up the arrays with all mapped variables.
10660 MapInfosOrErrorTy Info = GenMapInfoCB(Builder.saveIP(), PtrPHI, BeginIn);
10661 if (!Info)
10662 return Info.takeError();
10663
10664 // Call the runtime API __tgt_mapper_num_components to get the number of
10665 // pre-existing components.
10666 Value *OffloadingArgs[] = {MapperHandle};
10667 Value *PreviousSize = createRuntimeFunctionCall(
10668 Callee: getOrCreateRuntimeFunction(M, FnID: OMPRTL___tgt_mapper_num_components),
10669 Args: OffloadingArgs);
10670 Value *ShiftedPreviousSize =
10671 Builder.CreateShl(LHS: PreviousSize, RHS: Builder.getInt64(C: getFlagMemberOffset()));
10672
10673 // Fill up the runtime mapper handle for all components.
10674 for (unsigned I = 0; I < Info->BasePointers.size(); ++I) {
10675 Value *CurBaseArg = Info->BasePointers[I];
10676 Value *CurBeginArg = Info->Pointers[I];
10677 Value *CurSizeArg = Info->Sizes[I];
10678 Value *CurNameArg = Info->Names.size()
10679 ? Info->Names[I]
10680 : Constant::getNullValue(Ty: Builder.getPtrTy());
10681
10682 Value *OriMapType = Builder.getInt64(
10683 C: static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10684 Info->Types[I]));
10685 auto RawType =
10686 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10687 Info->Types[I]);
10688 constexpr uint64_t MemberOfMask =
10689 static_cast<uint64_t>(OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);
10690 constexpr uint64_t AttachBit =
10691 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10692 OpenMPOffloadMappingFlags::OMP_MAP_ATTACH);
10693
10694 // Add MEMBER_OF (ShiftedPreviousSize) to group this sub-map with the
10695 // current array element (N = __tgt_mapper_num_components() at loop body
10696 // start).
10697 //
10698 // Example 1:
10699 // struct S { int x; int *p; };
10700 //
10701 // mapper: #pragma omp declare mapper(id: S s) map(s.x, s.p[0:10])
10702 // use: S arr[2]; ... map(arr)
10703 // entries per element:
10704 //
10705 // &arr[i], &arr[i].x, sizeof(int), MEMBER_OF(N)|TO|FROM
10706 // &arr[i].p[0], &arr[i].p[0], 10*sizeof(int), TO|FROM (*)
10707 // &arr[i].p, &arr[i].p[0], sizeof(int*), ATTACH (**)
10708 //
10709 // Example 2:
10710 // struct S1 { int x; int y; };
10711 // struct S2 { int z; S1 *s1p; };
10712 //
10713 // mapper: #pragma omp declare mapper(S2 s2) map(s2.z, s2.s1p->x,
10714 // s2.s1p->y)
10715 // use: S2 arr[2]; ... map(arr)
10716 // entries per element:
10717 //
10718 // &arr[i], &arr[i].z, sizeof(int), MEMBER_OF(N)|TO|FROM
10719 // &arr[i].s1p[0], &arr[i].s1p->x, sizeof(s1p->x..y), ALLOC (*)
10720 // &arr[i].s1p[0], &arr[i].s1p->x, 4, MEMBER_OF(N+2)|TO|FROM (*)(***)
10721 // &arr[i].s1p[0], &arr[i].s1p->y, 4, MEMBER_OF(N+2)|TO|FROM (*)(***)
10722 // &arr[i].s1p, &arr[i].s1p->x, sizeof(ptr), ATTACH (**)
10723 //
10724 // x/y carry inner MEMBER_OF(2)
10725 // which is shifted by N to become MEMBER_OF(N+2).
10726 //
10727 // HasAttachPtr is set on all of the s1p entries except the ATTACH one:
10728 // the combined ALLOC entry for the s1p->x..y block, and the individual
10729 // x/y entries that are MEMBER_OF that block, all describe storage
10730 // reached through the attach ptr arr[i].s1p.
10731 //
10732 // Entries of the following kinds do NOT receive a new outer MEMBER_OF
10733 // linking them to the parent struct:
10734 //
10735 // * (*) Entries with HasAttachPtr: they represent pointee data that
10736 // occupies a different storage block than the struct being mapped, so
10737 // they are not a member of it. They may still be MEMBER_OF an entry
10738 // within that pointee block, in which case those pre-existing bits are
10739 // shifted -- see (***).
10740 // * (**) ATTACH entries: they are not a member of anything — they just
10741 // link a ptr to its ptee.
10742 // * All entries when PreserveMemberOfFlags is set (the Flang/MLIR path):
10743 // its pre-shaped entries already carry their final MEMBER_OF bits.
10744 // TODO: set HasAttachPtr from Flang for entries whose storage is the
10745 // pointee's (e.g. s%p(0:10)) and drop PreserveMemberOfFlags in favor of
10746 // it.
10747 //
10748 // (***) If such an entry already has its own MEMBER_OF bits (e.g. the
10749 // s1p->x/y entries above), those bits are still shifted by N.
10750 Value *MemberMapType;
10751 if (PreserveMemberOfFlags || (RawType & AttachBit) ||
10752 Info->HasAttachPtr[I]) {
10753 if (RawType & MemberOfMask)
10754 MemberMapType = Builder.CreateNUWAdd(LHS: OriMapType, RHS: ShiftedPreviousSize);
10755 else
10756 MemberMapType = OriMapType;
10757 } else {
10758 MemberMapType = Builder.CreateNUWAdd(LHS: OriMapType, RHS: ShiftedPreviousSize);
10759 }
10760
10761 // Combine the map type inherited from user-defined mapper with that
10762 // specified in the program. According to the OMP_MAP_TO and OMP_MAP_FROM
10763 // bits of the \a MapType, which is the input argument of the mapper
10764 // function, the following code will set the OMP_MAP_TO and OMP_MAP_FROM
10765 // bits of MemberMapType.
10766 // [OpenMP 5.0], 1.2.6. map-type decay.
10767 // | alloc | to | from | tofrom | release | delete
10768 // ----------------------------------------------------------
10769 // alloc | alloc | alloc | alloc | alloc | release | delete
10770 // to | alloc | to | alloc | to | release | delete
10771 // from | alloc | alloc | from | from | release | delete
10772 // tofrom | alloc | to | from | tofrom | release | delete
10773 Value *LeftToFrom = Builder.CreateAnd(
10774 LHS: MapType,
10775 RHS: Builder.getInt64(
10776 C: static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10777 OpenMPOffloadMappingFlags::OMP_MAP_TO |
10778 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
10779 BasicBlock *AllocBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp.type.alloc");
10780 BasicBlock *AllocElseBB =
10781 BasicBlock::Create(Context&: M.getContext(), Name: "omp.type.alloc.else");
10782 BasicBlock *ToBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp.type.to");
10783 BasicBlock *ToElseBB =
10784 BasicBlock::Create(Context&: M.getContext(), Name: "omp.type.to.else");
10785 BasicBlock *FromBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp.type.from");
10786 BasicBlock *EndBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp.type.end");
10787 Value *IsAlloc = Builder.CreateIsNull(Arg: LeftToFrom);
10788 Builder.CreateCondBr(Cond: IsAlloc, True: AllocBB, False: AllocElseBB);
10789 // In case of alloc, clear OMP_MAP_TO and OMP_MAP_FROM.
10790 emitBlock(BB: AllocBB, CurFn: MapperFn);
10791 Value *AllocMapType = Builder.CreateAnd(
10792 LHS: MemberMapType,
10793 RHS: Builder.getInt64(
10794 C: ~static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10795 OpenMPOffloadMappingFlags::OMP_MAP_TO |
10796 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
10797 Builder.CreateBr(Dest: EndBB);
10798 emitBlock(BB: AllocElseBB, CurFn: MapperFn);
10799 Value *IsTo = Builder.CreateICmpEQ(
10800 LHS: LeftToFrom,
10801 RHS: Builder.getInt64(
10802 C: static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10803 OpenMPOffloadMappingFlags::OMP_MAP_TO)));
10804 Builder.CreateCondBr(Cond: IsTo, True: ToBB, False: ToElseBB);
10805 // In case of to, clear OMP_MAP_FROM.
10806 emitBlock(BB: ToBB, CurFn: MapperFn);
10807 Value *ToMapType = Builder.CreateAnd(
10808 LHS: MemberMapType,
10809 RHS: Builder.getInt64(
10810 C: ~static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10811 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
10812 Builder.CreateBr(Dest: EndBB);
10813 emitBlock(BB: ToElseBB, CurFn: MapperFn);
10814 Value *IsFrom = Builder.CreateICmpEQ(
10815 LHS: LeftToFrom,
10816 RHS: Builder.getInt64(
10817 C: static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10818 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
10819 Builder.CreateCondBr(Cond: IsFrom, True: FromBB, False: EndBB);
10820 // In case of from, clear OMP_MAP_TO.
10821 emitBlock(BB: FromBB, CurFn: MapperFn);
10822 Value *FromMapType = Builder.CreateAnd(
10823 LHS: MemberMapType,
10824 RHS: Builder.getInt64(
10825 C: ~static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10826 OpenMPOffloadMappingFlags::OMP_MAP_TO)));
10827 // In case of tofrom, do nothing.
10828 emitBlock(BB: EndBB, CurFn: MapperFn);
10829 LastBB = EndBB;
10830 PHINode *CurMapType =
10831 Builder.CreatePHI(Ty: Builder.getInt64Ty(), NumReservedValues: 4, Name: "omp.maptype");
10832 CurMapType->addIncoming(V: AllocMapType, BB: AllocBB);
10833 CurMapType->addIncoming(V: ToMapType, BB: ToBB);
10834 CurMapType->addIncoming(V: FromMapType, BB: FromBB);
10835 CurMapType->addIncoming(V: MemberMapType, BB: ToElseBB);
10836
10837 // Propagate map-type-modifying bits from the outer map clause to each map
10838 // inserted by the mapper.
10839 //
10840 // OpenMP 6.0:281:34: The effect of the mapper modifier is to remove the
10841 // list item from the map clause and to apply the clauses specified in the
10842 // declared mapper to the construct on which the map clause appears...
10843 // If any modifier with the map-type-modifying property appears in the map
10844 // clause then the effect is as if that modifier appears in each map clause
10845 // specified in the declared mapper.
10846 //
10847 // Map-type-modifying bits: ALWAYS, DELETE, CLOSE, PRESENT.
10848 //
10849 // ALWAYS/DELETE/CLOSE are propagated to every (non-ATTACH) entry.
10850 //
10851 // PRESENT is propagated only to entries that have an attach ptr
10852 // (HasAttachPtr): the pointee data, which occupies a different storage
10853 // block than the struct being mapped and so is not covered by the
10854 // present-check on the struct's own storage. A present modifier on the
10855 // outer clause must still require that pointee to be present on the device.
10856 //
10857 // This is gated on \p PropagatePresentToPointee (set by callers only for
10858 // OpenMP >= 6.0). Before 6.0 the present modifier is treated as not
10859 // applying to the pointee: the spec committee confirmed the divergence
10860 // between the present "motion" modifier (to/from) and the present map-type
10861 // modifier (map) was unintentional, to be fixed as an OpenMP 6.0 erratum,
10862 // so for 5.2 present is ignored for the pointee for both map and to/from.
10863 //
10864 // TODO: PRESENT should also be propagated to the struct's own members
10865 // (e.g. the s.x, s.y of map(present, mapper(id): s)) so that an absent
10866 // member triggers the present-check. We cannot do that yet: while pointer
10867 // members are mapped with PTR_AND_OBJ, a single combined entry allocates
10868 // the whole struct (including the pointer's storage), so propagating
10869 // PRESENT to it would wrongly require the pointer's pointee to be present.
10870 // Enable member propagation once Clang stops emitting PTR_AND_OBJ and uses
10871 // attach-style maps throughout.
10872 uint64_t ModifierBits =
10873 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10874 OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS |
10875 OpenMPOffloadMappingFlags::OMP_MAP_DELETE |
10876 OpenMPOffloadMappingFlags::OMP_MAP_CLOSE);
10877 if (PropagatePresentToPointee && Info->HasAttachPtr[I])
10878 ModifierBits |=
10879 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10880 OpenMPOffloadMappingFlags::OMP_MAP_PRESENT);
10881 Value *ImportedModifierBits =
10882 Builder.CreateAnd(LHS: MapType, RHS: Builder.getInt64(C: ModifierBits));
10883 Value *CurMapTypeWithModifiers = Builder.CreateOr(
10884 LHS: CurMapType, RHS: ImportedModifierBits, Name: "omp.maptype.with.modifiers");
10885
10886 // ATTACH entries must not receive map-type-modifying bits: ATTACH|ALWAYS is
10887 // reserved for the attach(always) map-type modifier, and other modifier
10888 // bits (DELETE, CLOSE) have no meaning for an ATTACH entry.
10889 Value *FinalMapType =
10890 (RawType & AttachBit) ? CurMapType : CurMapTypeWithModifiers;
10891
10892 Value *OffloadingArgs[] = {MapperHandle, CurBaseArg, CurBeginArg,
10893 CurSizeArg, FinalMapType, CurNameArg};
10894
10895 auto ChildMapperFn = CustomMapperCB(I);
10896 if (!ChildMapperFn)
10897 return ChildMapperFn.takeError();
10898 if (*ChildMapperFn) {
10899 // Call the corresponding mapper function.
10900 createRuntimeFunctionCall(Callee: *ChildMapperFn, Args: OffloadingArgs)
10901 ->setDoesNotThrow();
10902 } else {
10903 // Call the runtime API __tgt_push_mapper_component to fill up the runtime
10904 // data structure.
10905 createRuntimeFunctionCall(
10906 Callee: getOrCreateRuntimeFunction(M, FnID: OMPRTL___tgt_push_mapper_component),
10907 Args: OffloadingArgs);
10908 }
10909 }
10910
10911 // Update the pointer to point to the next element that needs to be mapped,
10912 // and check whether we have mapped all elements.
10913 Value *PtrNext = Builder.CreateConstGEP1_32(Ty: ElemTy, Ptr: PtrPHI, /*Idx0=*/1,
10914 Name: "omp.arraymap.next");
10915 PtrPHI->addIncoming(V: PtrNext, BB: LastBB);
10916 Value *IsDone = Builder.CreateICmpEQ(LHS: PtrNext, RHS: PtrEnd, Name: "omp.arraymap.isdone");
10917 BasicBlock *ExitBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp.arraymap.exit");
10918 Builder.CreateCondBr(Cond: IsDone, True: ExitBB, False: BodyBB);
10919
10920 emitBlock(BB: ExitBB, CurFn: MapperFn);
10921 // Emit array deletion if this is an array section and \p MapType indicates
10922 // that deletion is required.
10923 emitUDMapperArrayInitOrDel(MapperFn, MapperHandle, Base: BaseIn, Begin: BeginIn, Size,
10924 MapType, MapName, ElementSize, ExitBB: DoneBB,
10925 /*IsInit=*/false);
10926
10927 // Emit the function exit block.
10928 emitBlock(BB: DoneBB, CurFn: MapperFn, /*IsFinished=*/true);
10929
10930 Builder.CreateRetVoid();
10931 return MapperFn;
10932}
10933
10934Error OpenMPIRBuilder::emitOffloadingArrays(
10935 InsertPointTy AllocaIP, InsertPointTy CodeGenIP, MapInfosTy &CombinedInfo,
10936 TargetDataInfo &Info, CustomMapperCallbackTy CustomMapperCB,
10937 bool IsNonContiguous,
10938 function_ref<void(unsigned int, Value *)> DeviceAddrCB) {
10939
10940 // Reset the array information.
10941 Info.clearArrayInfo();
10942 Info.NumberOfPtrs = CombinedInfo.BasePointers.size();
10943
10944 if (Info.NumberOfPtrs == 0)
10945 return Error::success();
10946
10947 Builder.restoreIP(IP: AllocaIP);
10948 // Detect if we have any capture size requiring runtime evaluation of the
10949 // size so that a constant array could be eventually used.
10950 ArrayType *PointerArrayType =
10951 ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: Info.NumberOfPtrs);
10952
10953 Info.RTArgs.BasePointersArray = Builder.CreateAlloca(
10954 Ty: PointerArrayType, /* ArraySize = */ nullptr, Name: ".offload_baseptrs");
10955
10956 Info.RTArgs.PointersArray = Builder.CreateAlloca(
10957 Ty: PointerArrayType, /* ArraySize = */ nullptr, Name: ".offload_ptrs");
10958 AllocaInst *MappersArray = Builder.CreateAlloca(
10959 Ty: PointerArrayType, /* ArraySize = */ nullptr, Name: ".offload_mappers");
10960 Info.RTArgs.MappersArray = MappersArray;
10961
10962 // If we don't have any VLA types or other types that require runtime
10963 // evaluation, we can use a constant array for the map sizes, otherwise we
10964 // need to fill up the arrays as we do for the pointers.
10965 Type *Int64Ty = Builder.getInt64Ty();
10966 SmallVector<Constant *> ConstSizes(CombinedInfo.Sizes.size(),
10967 ConstantInt::get(Ty: Int64Ty, V: 0));
10968 SmallBitVector RuntimeSizes(CombinedInfo.Sizes.size());
10969 for (unsigned I = 0, E = CombinedInfo.Sizes.size(); I < E; ++I) {
10970 bool IsNonContigEntry =
10971 IsNonContiguous &&
10972 (static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10973 CombinedInfo.Types[I] &
10974 OpenMPOffloadMappingFlags::OMP_MAP_NON_CONTIG) != 0);
10975 // For NON_CONTIG entries, ArgSizes stores the dimension count (number of
10976 // descriptor_dim records), not the byte size.
10977 if (IsNonContigEntry) {
10978 assert(I < CombinedInfo.NonContigInfo.Dims.size() &&
10979 "Index must be in-bounds for NON_CONTIG Dims array");
10980 const uint64_t DimCount = CombinedInfo.NonContigInfo.Dims[I];
10981 assert(DimCount > 0 && "NON_CONTIG DimCount must be > 0");
10982 ConstSizes[I] = ConstantInt::get(Ty: Int64Ty, V: DimCount);
10983 continue;
10984 }
10985 if (auto *CI = dyn_cast<Constant>(Val: CombinedInfo.Sizes[I])) {
10986 if (!isa<ConstantExpr>(Val: CI) && !isa<GlobalValue>(Val: CI)) {
10987 ConstSizes[I] = CI;
10988 continue;
10989 }
10990 }
10991 RuntimeSizes.set(I);
10992 }
10993
10994 if (RuntimeSizes.all()) {
10995 ArrayType *SizeArrayType = ArrayType::get(ElementType: Int64Ty, NumElements: Info.NumberOfPtrs);
10996 Info.RTArgs.SizesArray = Builder.CreateAlloca(
10997 Ty: SizeArrayType, /* ArraySize = */ nullptr, Name: ".offload_sizes");
10998 restoreIPandDebugLoc(Builder, IP: CodeGenIP);
10999 } else {
11000 auto *SizesArrayInit = ConstantArray::get(
11001 T: ArrayType::get(ElementType: Int64Ty, NumElements: ConstSizes.size()), V: ConstSizes);
11002 std::string Name = createPlatformSpecificName(Parts: {"offload_sizes"});
11003 auto *SizesArrayGbl =
11004 new GlobalVariable(M, SizesArrayInit->getType(), /*isConstant=*/true,
11005 GlobalValue::PrivateLinkage, SizesArrayInit, Name);
11006 SizesArrayGbl->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
11007
11008 if (!RuntimeSizes.any()) {
11009 Info.RTArgs.SizesArray = SizesArrayGbl;
11010 } else {
11011 unsigned IndexSize = M.getDataLayout().getIndexSizeInBits(AS: 0);
11012 Align OffloadSizeAlign = M.getDataLayout().getABIIntegerTypeAlignment(BitWidth: 64);
11013 ArrayType *SizeArrayType = ArrayType::get(ElementType: Int64Ty, NumElements: Info.NumberOfPtrs);
11014 AllocaInst *Buffer = Builder.CreateAlloca(
11015 Ty: SizeArrayType, /* ArraySize = */ nullptr, Name: ".offload_sizes");
11016 Buffer->setAlignment(OffloadSizeAlign);
11017 restoreIPandDebugLoc(Builder, IP: CodeGenIP);
11018 Builder.CreateMemCpy(
11019 Dst: Buffer, DstAlign: M.getDataLayout().getPrefTypeAlign(Ty: Buffer->getType()),
11020 Src: SizesArrayGbl, SrcAlign: OffloadSizeAlign,
11021 Size: Builder.getIntN(
11022 N: IndexSize,
11023 C: Buffer->getAllocationSize(DL: M.getDataLayout())->getFixedValue()));
11024
11025 Info.RTArgs.SizesArray = Buffer;
11026 }
11027 restoreIPandDebugLoc(Builder, IP: CodeGenIP);
11028 }
11029
11030 // The map types are always constant so we don't need to generate code to
11031 // fill arrays. Instead, we create an array constant.
11032 SmallVector<uint64_t, 4> Mapping;
11033 for (auto mapFlag : CombinedInfo.Types)
11034 Mapping.push_back(
11035 Elt: static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
11036 mapFlag));
11037 std::string MaptypesName = createPlatformSpecificName(Parts: {"offload_maptypes"});
11038 auto *MapTypesArrayGbl = createOffloadMaptypes(Mappings&: Mapping, VarName: MaptypesName);
11039 Info.RTArgs.MapTypesArray = MapTypesArrayGbl;
11040
11041 // The information types are only built if provided.
11042 if (!CombinedInfo.Names.empty()) {
11043 auto *MapNamesArrayGbl = createOffloadMapnames(
11044 Names&: CombinedInfo.Names, VarName: createPlatformSpecificName(Parts: {"offload_mapnames"}));
11045 Info.RTArgs.MapNamesArray = MapNamesArrayGbl;
11046 Info.EmitDebug = true;
11047 } else {
11048 Info.RTArgs.MapNamesArray =
11049 Constant::getNullValue(Ty: PointerType::getUnqual(C&: Builder.getContext()));
11050 Info.EmitDebug = false;
11051 }
11052
11053 // If there's a present map type modifier, it must not be applied to the end
11054 // of a region, so generate a separate map type array in that case.
11055 if (Info.separateBeginEndCalls()) {
11056 bool EndMapTypesDiffer = false;
11057 for (uint64_t &Type : Mapping) {
11058 if (Type & static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
11059 OpenMPOffloadMappingFlags::OMP_MAP_PRESENT)) {
11060 Type &= ~static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
11061 OpenMPOffloadMappingFlags::OMP_MAP_PRESENT);
11062 EndMapTypesDiffer = true;
11063 }
11064 }
11065 if (EndMapTypesDiffer) {
11066 MapTypesArrayGbl = createOffloadMaptypes(Mappings&: Mapping, VarName: MaptypesName);
11067 Info.RTArgs.MapTypesArrayEnd = MapTypesArrayGbl;
11068 }
11069 }
11070
11071 PointerType *PtrTy = Builder.getPtrTy();
11072 for (unsigned I = 0; I < Info.NumberOfPtrs; ++I) {
11073 Value *BPVal = CombinedInfo.BasePointers[I];
11074 Value *BP = Builder.CreateConstInBoundsGEP2_32(
11075 Ty: ArrayType::get(ElementType: PtrTy, NumElements: Info.NumberOfPtrs), Ptr: Info.RTArgs.BasePointersArray,
11076 Idx0: 0, Idx1: I);
11077 Builder.CreateAlignedStore(Val: BPVal, Ptr: BP,
11078 Align: M.getDataLayout().getPrefTypeAlign(Ty: PtrTy));
11079
11080 if (Info.requiresDevicePointerInfo()) {
11081 if (CombinedInfo.DevicePointers[I] == DeviceInfoTy::Pointer) {
11082 CodeGenIP = Builder.saveIP();
11083 Builder.restoreIP(IP: AllocaIP);
11084 Info.DevicePtrInfoMap[BPVal] = {BP, Builder.CreateAlloca(Ty: PtrTy)};
11085 restoreIPandDebugLoc(Builder, IP: CodeGenIP);
11086 if (DeviceAddrCB)
11087 DeviceAddrCB(I, Info.DevicePtrInfoMap[BPVal].second);
11088 } else if (CombinedInfo.DevicePointers[I] == DeviceInfoTy::Address) {
11089 Info.DevicePtrInfoMap[BPVal] = {BP, BP};
11090 if (DeviceAddrCB)
11091 DeviceAddrCB(I, BP);
11092 }
11093 }
11094
11095 Value *PVal = CombinedInfo.Pointers[I];
11096 Value *P = Builder.CreateConstInBoundsGEP2_32(
11097 Ty: ArrayType::get(ElementType: PtrTy, NumElements: Info.NumberOfPtrs), Ptr: Info.RTArgs.PointersArray, Idx0: 0,
11098 Idx1: I);
11099 // TODO: Check alignment correct.
11100 Builder.CreateAlignedStore(Val: PVal, Ptr: P,
11101 Align: M.getDataLayout().getPrefTypeAlign(Ty: PtrTy));
11102
11103 if (RuntimeSizes.test(Idx: I)) {
11104 Value *S = Builder.CreateConstInBoundsGEP2_32(
11105 Ty: ArrayType::get(ElementType: Int64Ty, NumElements: Info.NumberOfPtrs), Ptr: Info.RTArgs.SizesArray,
11106 /*Idx0=*/0,
11107 /*Idx1=*/I);
11108 Builder.CreateAlignedStore(Val: Builder.CreateIntCast(V: CombinedInfo.Sizes[I],
11109 DestTy: Int64Ty,
11110 /*isSigned=*/true),
11111 Ptr: S, Align: M.getDataLayout().getPrefTypeAlign(Ty: PtrTy));
11112 }
11113 // Fill up the mapper array.
11114 unsigned IndexSize = M.getDataLayout().getIndexSizeInBits(AS: 0);
11115 Value *MFunc = ConstantPointerNull::get(T: PtrTy);
11116
11117 auto CustomMFunc = CustomMapperCB(I);
11118 if (!CustomMFunc)
11119 return CustomMFunc.takeError();
11120 if (*CustomMFunc)
11121 MFunc = Builder.CreatePointerCast(V: *CustomMFunc, DestTy: PtrTy);
11122
11123 Value *MAddr = Builder.CreateInBoundsGEP(
11124 Ty: PointerArrayType, Ptr: MappersArray,
11125 IdxList: {Builder.getIntN(N: IndexSize, C: 0), Builder.getIntN(N: IndexSize, C: I)});
11126 Builder.CreateAlignedStore(
11127 Val: MFunc, Ptr: MAddr, Align: M.getDataLayout().getPrefTypeAlign(Ty: MAddr->getType()));
11128 }
11129
11130 if (!IsNonContiguous || CombinedInfo.NonContigInfo.Offsets.empty() ||
11131 Info.NumberOfPtrs == 0)
11132 return Error::success();
11133 emitNonContiguousDescriptor(AllocaIP, CodeGenIP, CombinedInfo, Info);
11134 return Error::success();
11135}
11136
11137void OpenMPIRBuilder::emitBranch(BasicBlock *Target) {
11138 BasicBlock *CurBB = Builder.GetInsertBlock();
11139
11140 if (!CurBB || CurBB->hasTerminator()) {
11141 // If there is no insert point or the previous block is already
11142 // terminated, don't touch it.
11143 } else {
11144 // Otherwise, create a fall-through branch.
11145 Builder.CreateBr(Dest: Target);
11146 }
11147
11148 Builder.ClearInsertionPoint();
11149}
11150
11151void OpenMPIRBuilder::emitBlock(BasicBlock *BB, Function *CurFn,
11152 bool IsFinished) {
11153 BasicBlock *CurBB = Builder.GetInsertBlock();
11154
11155 // Fall out of the current block (if necessary).
11156 emitBranch(Target: BB);
11157
11158 if (IsFinished && BB->use_empty()) {
11159 BB->eraseFromParent();
11160 return;
11161 }
11162
11163 // Place the block after the current block, if possible, or else at
11164 // the end of the function.
11165 if (CurBB && CurBB->getParent())
11166 CurFn->insert(Position: std::next(x: CurBB->getIterator()), BB);
11167 else
11168 CurFn->insert(Position: CurFn->end(), BB);
11169 Builder.SetInsertPoint(BB);
11170}
11171
11172Error OpenMPIRBuilder::emitIfClause(Value *Cond, BodyGenCallbackTy ThenGen,
11173 BodyGenCallbackTy ElseGen,
11174 InsertPointTy AllocaIP,
11175 ArrayRef<BasicBlock *> DeallocBlocks) {
11176 // If the condition constant folds and can be elided, try to avoid emitting
11177 // the condition and the dead arm of the if/else.
11178 if (auto *CI = dyn_cast<ConstantInt>(Val: Cond)) {
11179 auto CondConstant = CI->getSExtValue();
11180 if (CondConstant)
11181 return ThenGen(AllocaIP, Builder.saveIP(), DeallocBlocks);
11182
11183 return ElseGen(AllocaIP, Builder.saveIP(), DeallocBlocks);
11184 }
11185
11186 Function *CurFn = Builder.GetInsertBlock()->getParent();
11187
11188 // Otherwise, the condition did not fold, or we couldn't elide it. Just
11189 // emit the conditional branch.
11190 BasicBlock *ThenBlock = BasicBlock::Create(Context&: M.getContext(), Name: "omp_if.then");
11191 BasicBlock *ElseBlock = BasicBlock::Create(Context&: M.getContext(), Name: "omp_if.else");
11192 BasicBlock *ContBlock = BasicBlock::Create(Context&: M.getContext(), Name: "omp_if.end");
11193 Builder.CreateCondBr(Cond, True: ThenBlock, False: ElseBlock);
11194 // Emit the 'then' code.
11195 emitBlock(BB: ThenBlock, CurFn);
11196 if (Error Err = ThenGen(AllocaIP, Builder.saveIP(), DeallocBlocks))
11197 return Err;
11198 emitBranch(Target: ContBlock);
11199 // Emit the 'else' code if present.
11200 // There is no need to emit line number for unconditional branch.
11201 emitBlock(BB: ElseBlock, CurFn);
11202 if (Error Err = ElseGen(AllocaIP, Builder.saveIP(), DeallocBlocks))
11203 return Err;
11204 // There is no need to emit line number for unconditional branch.
11205 emitBranch(Target: ContBlock);
11206 // Emit the continuation block for code after the if.
11207 emitBlock(BB: ContBlock, CurFn, /*IsFinished=*/true);
11208 return Error::success();
11209}
11210
11211bool OpenMPIRBuilder::checkAndEmitFlushAfterAtomic(
11212 const LocationDescription &Loc, llvm::AtomicOrdering AO, AtomicKind AK) {
11213 assert(!(AO == AtomicOrdering::NotAtomic ||
11214 AO == llvm::AtomicOrdering::Unordered) &&
11215 "Unexpected Atomic Ordering.");
11216
11217 bool Flush = false;
11218 llvm::AtomicOrdering FlushAO = AtomicOrdering::Monotonic;
11219
11220 switch (AK) {
11221 case Read:
11222 if (AO == AtomicOrdering::Acquire || AO == AtomicOrdering::AcquireRelease ||
11223 AO == AtomicOrdering::SequentiallyConsistent) {
11224 FlushAO = AtomicOrdering::Acquire;
11225 Flush = true;
11226 }
11227 break;
11228 case Write:
11229 case Compare:
11230 case Update:
11231 if (AO == AtomicOrdering::Release || AO == AtomicOrdering::AcquireRelease ||
11232 AO == AtomicOrdering::SequentiallyConsistent) {
11233 FlushAO = AtomicOrdering::Release;
11234 Flush = true;
11235 }
11236 break;
11237 case Capture:
11238 switch (AO) {
11239 case AtomicOrdering::Acquire:
11240 FlushAO = AtomicOrdering::Acquire;
11241 Flush = true;
11242 break;
11243 case AtomicOrdering::Release:
11244 FlushAO = AtomicOrdering::Release;
11245 Flush = true;
11246 break;
11247 case AtomicOrdering::AcquireRelease:
11248 case AtomicOrdering::SequentiallyConsistent:
11249 FlushAO = AtomicOrdering::AcquireRelease;
11250 Flush = true;
11251 break;
11252 default:
11253 // do nothing - leave silently.
11254 break;
11255 }
11256 }
11257
11258 if (Flush) {
11259 // Currently Flush RT call still doesn't take memory_ordering, so for when
11260 // that happens, this tries to do the resolution of which atomic ordering
11261 // to use with but issue the flush call
11262 // TODO: pass `FlushAO` after memory ordering support is added
11263 (void)FlushAO;
11264 emitFlush(Loc);
11265 }
11266
11267 // for AO == AtomicOrdering::Monotonic and all other case combinations
11268 // do nothing
11269 return Flush;
11270}
11271
11272OpenMPIRBuilder::InsertPointTy
11273OpenMPIRBuilder::createAtomicRead(const LocationDescription &Loc,
11274 AtomicOpValue &X, AtomicOpValue &V,
11275 AtomicOrdering AO, InsertPointTy AllocaIP) {
11276 if (!updateToLocation(Loc))
11277 return Loc.IP;
11278
11279 assert(X.Var->getType()->isPointerTy() &&
11280 "OMP Atomic expects a pointer to target memory");
11281 Type *XElemTy = X.ElemTy;
11282 assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||
11283 XElemTy->isPointerTy() || XElemTy->isStructTy()) &&
11284 "OMP atomic read expected a scalar type");
11285
11286 Value *XRead = nullptr;
11287
11288 if (XElemTy->isIntegerTy()) {
11289 LoadInst *XLD =
11290 Builder.CreateLoad(Ty: XElemTy, Ptr: X.Var, isVolatile: X.IsVolatile, Name: "omp.atomic.read");
11291 XLD->setAtomic(Ordering: AO);
11292 XRead = cast<Value>(Val: XLD);
11293 } else if (XElemTy->isStructTy()) {
11294 // FIXME: Add checks to ensure __atomic_load is emitted iff the
11295 // target does not support `atomicrmw` of the size of the struct
11296 LoadInst *OldVal = Builder.CreateLoad(Ty: XElemTy, Ptr: X.Var, Name: "omp.atomic.read");
11297 OldVal->setAtomic(Ordering: AO);
11298 const DataLayout &DL = OldVal->getModule()->getDataLayout();
11299 unsigned LoadSize = DL.getTypeStoreSize(Ty: XElemTy);
11300 OpenMPIRBuilder::AtomicInfo atomicInfo(
11301 &Builder, XElemTy, LoadSize * 8, LoadSize * 8, OldVal->getAlign(),
11302 OldVal->getAlign(), true /* UseLibcall */, AllocaIP, X.Var);
11303 auto AtomicLoadRes = atomicInfo.EmitAtomicLoadLibcall(AO);
11304 XRead = AtomicLoadRes.first;
11305 OldVal->eraseFromParent();
11306 } else {
11307 // We need to perform atomic op as integer
11308 IntegerType *IntCastTy =
11309 IntegerType::get(C&: M.getContext(), NumBits: XElemTy->getScalarSizeInBits());
11310 LoadInst *XLoad =
11311 Builder.CreateLoad(Ty: IntCastTy, Ptr: X.Var, isVolatile: X.IsVolatile, Name: "omp.atomic.load");
11312 XLoad->setAtomic(Ordering: AO);
11313 if (XElemTy->isFloatingPointTy()) {
11314 XRead = Builder.CreateBitCast(V: XLoad, DestTy: XElemTy, Name: "atomic.flt.cast");
11315 } else {
11316 XRead = Builder.CreateIntToPtr(V: XLoad, DestTy: XElemTy, Name: "atomic.ptr.cast");
11317 }
11318 }
11319 checkAndEmitFlushAfterAtomic(Loc, AO, AK: AtomicKind::Read);
11320 Builder.CreateStore(Val: XRead, Ptr: V.Var, isVolatile: V.IsVolatile);
11321 return Builder.saveIP();
11322}
11323
11324OpenMPIRBuilder::InsertPointTy
11325OpenMPIRBuilder::createAtomicWrite(const LocationDescription &Loc,
11326 AtomicOpValue &X, Value *Expr,
11327 AtomicOrdering AO, InsertPointTy AllocaIP) {
11328 if (!updateToLocation(Loc))
11329 return Loc.IP;
11330
11331 assert(X.Var->getType()->isPointerTy() &&
11332 "OMP Atomic expects a pointer to target memory");
11333 Type *XElemTy = X.ElemTy;
11334 assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||
11335 XElemTy->isPointerTy() || XElemTy->isStructTy()) &&
11336 "OMP atomic write expected a scalar type");
11337
11338 if (XElemTy->isIntegerTy()) {
11339 StoreInst *XSt = Builder.CreateStore(Val: Expr, Ptr: X.Var, isVolatile: X.IsVolatile);
11340 XSt->setAtomic(Ordering: AO);
11341 } else if (XElemTy->isStructTy()) {
11342 LoadInst *OldVal = Builder.CreateLoad(Ty: XElemTy, Ptr: X.Var, Name: "omp.atomic.read");
11343 const DataLayout &DL = OldVal->getModule()->getDataLayout();
11344 unsigned LoadSize = DL.getTypeStoreSize(Ty: XElemTy);
11345 OpenMPIRBuilder::AtomicInfo atomicInfo(
11346 &Builder, XElemTy, LoadSize * 8, LoadSize * 8, OldVal->getAlign(),
11347 OldVal->getAlign(), true /* UseLibcall */, AllocaIP, X.Var);
11348 atomicInfo.EmitAtomicStoreLibcall(AO, Source: Expr);
11349 OldVal->eraseFromParent();
11350 } else {
11351 // We need to bitcast and perform atomic op as integers
11352 IntegerType *IntCastTy =
11353 IntegerType::get(C&: M.getContext(), NumBits: XElemTy->getScalarSizeInBits());
11354 Value *ExprCast =
11355 Builder.CreateBitCast(V: Expr, DestTy: IntCastTy, Name: "atomic.src.int.cast");
11356 StoreInst *XSt = Builder.CreateStore(Val: ExprCast, Ptr: X.Var, isVolatile: X.IsVolatile);
11357 XSt->setAtomic(Ordering: AO);
11358 }
11359
11360 checkAndEmitFlushAfterAtomic(Loc, AO, AK: AtomicKind::Write);
11361 return Builder.saveIP();
11362}
11363
11364OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createAtomicUpdate(
11365 const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X,
11366 Value *Expr, AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp,
11367 AtomicUpdateCallbackTy &UpdateOp, bool IsXBinopExpr,
11368 bool IsIgnoreDenormalMode, bool IsFineGrainedMemory, bool IsRemoteMemory) {
11369 assert(!isConflictIP(Loc.IP, AllocaIP) && "IPs must not be ambiguous");
11370 if (!updateToLocation(Loc))
11371 return Loc.IP;
11372
11373 LLVM_DEBUG({
11374 Type *XTy = X.Var->getType();
11375 assert(XTy->isPointerTy() &&
11376 "OMP Atomic expects a pointer to target memory");
11377 Type *XElemTy = X.ElemTy;
11378 assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||
11379 XElemTy->isPointerTy() || XElemTy->isStructTy()) &&
11380 "OMP atomic update expected a scalar or struct type");
11381 assert((RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) &&
11382 (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) &&
11383 "OpenMP atomic does not support LT or GT operations");
11384 });
11385
11386 Expected<std::pair<Value *, Value *>> AtomicResult = emitAtomicUpdate(
11387 AllocaIP, X: X.Var, XElemTy: X.ElemTy, Expr, AO, RMWOp, UpdateOp, VolatileX: X.IsVolatile,
11388 IsXBinopExpr, IsIgnoreDenormalMode, IsFineGrainedMemory, IsRemoteMemory);
11389 if (!AtomicResult)
11390 return AtomicResult.takeError();
11391 checkAndEmitFlushAfterAtomic(Loc, AO, AK: AtomicKind::Update);
11392 return Builder.saveIP();
11393}
11394
11395// FIXME: Duplicating AtomicExpand
11396Value *OpenMPIRBuilder::emitRMWOpAsInstruction(Value *Src1, Value *Src2,
11397 AtomicRMWInst::BinOp RMWOp) {
11398 switch (RMWOp) {
11399 case AtomicRMWInst::Add:
11400 return Builder.CreateAdd(LHS: Src1, RHS: Src2);
11401 case AtomicRMWInst::Sub:
11402 return Builder.CreateSub(LHS: Src1, RHS: Src2);
11403 case AtomicRMWInst::And:
11404 return Builder.CreateAnd(LHS: Src1, RHS: Src2);
11405 case AtomicRMWInst::Nand:
11406 return Builder.CreateNeg(V: Builder.CreateAnd(LHS: Src1, RHS: Src2));
11407 case AtomicRMWInst::Or:
11408 return Builder.CreateOr(LHS: Src1, RHS: Src2);
11409 case AtomicRMWInst::Xor:
11410 return Builder.CreateXor(LHS: Src1, RHS: Src2);
11411 case AtomicRMWInst::Xchg:
11412 case AtomicRMWInst::FAdd:
11413 case AtomicRMWInst::FSub:
11414 case AtomicRMWInst::BAD_BINOP:
11415 case AtomicRMWInst::Max:
11416 case AtomicRMWInst::Min:
11417 case AtomicRMWInst::UMax:
11418 case AtomicRMWInst::UMin:
11419 case AtomicRMWInst::FMax:
11420 case AtomicRMWInst::FMin:
11421 case AtomicRMWInst::FMaximum:
11422 case AtomicRMWInst::FMinimum:
11423 case AtomicRMWInst::FMaximumNum:
11424 case AtomicRMWInst::FMinimumNum:
11425 case AtomicRMWInst::UIncWrap:
11426 case AtomicRMWInst::UDecWrap:
11427 case AtomicRMWInst::USubCond:
11428 case AtomicRMWInst::USubSat:
11429 llvm_unreachable("Unsupported atomic update operation");
11430 }
11431 llvm_unreachable("Unsupported atomic update operation");
11432}
11433
11434static AtomicOrdering TransformReleaseAcquireRelease(AtomicOrdering AO) {
11435 // Loads cannot use Release or AcquireRelease ordering. This load is
11436 // just the initial value for the cmpxchg loop; the cmpxchg itself
11437 // retains the original ordering.
11438 AtomicOrdering LoadAO = AO;
11439
11440 if (AO == AtomicOrdering::Release) {
11441 LoadAO = AtomicOrdering::Monotonic;
11442 } else if (AO == AtomicOrdering::AcquireRelease) {
11443 LoadAO = AtomicOrdering::Acquire;
11444 }
11445
11446 return LoadAO;
11447}
11448
11449Expected<std::pair<Value *, Value *>> OpenMPIRBuilder::emitAtomicUpdate(
11450 InsertPointTy AllocaIP, Value *X, Type *XElemTy, Value *Expr,
11451 AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp,
11452 AtomicUpdateCallbackTy &UpdateOp, bool VolatileX, bool IsXBinopExpr,
11453 bool IsIgnoreDenormalMode, bool IsFineGrainedMemory, bool IsRemoteMemory) {
11454 // TODO: handle the case where XElemTy is not byte-sized or not a power of 2.
11455 bool emitRMWOp = false;
11456 switch (RMWOp) {
11457 case AtomicRMWInst::Add:
11458 case AtomicRMWInst::And:
11459 case AtomicRMWInst::Nand:
11460 case AtomicRMWInst::Or:
11461 case AtomicRMWInst::Xor:
11462 case AtomicRMWInst::Xchg:
11463 emitRMWOp = XElemTy;
11464 break;
11465 case AtomicRMWInst::Sub:
11466 emitRMWOp = (IsXBinopExpr && XElemTy);
11467 break;
11468 default:
11469 emitRMWOp = false;
11470 }
11471 emitRMWOp &= XElemTy->isIntegerTy();
11472
11473 std::pair<Value *, Value *> Res;
11474 if (emitRMWOp) {
11475 AtomicRMWInst *RMWInst =
11476 Builder.CreateAtomicRMW(Op: RMWOp, Ptr: X, Val: Expr, Align: llvm::MaybeAlign(), Ordering: AO);
11477 if (T.isAMDGPU()) {
11478 if (IsIgnoreDenormalMode)
11479 RMWInst->setMetadata(Kind: "amdgpu.ignore.denormal.mode",
11480 Node: llvm::MDNode::get(Context&: Builder.getContext(), MDs: {}));
11481 if (!IsFineGrainedMemory)
11482 RMWInst->setMetadata(Kind: "amdgpu.no.fine.grained.memory",
11483 Node: llvm::MDNode::get(Context&: Builder.getContext(), MDs: {}));
11484 if (!IsRemoteMemory)
11485 RMWInst->setMetadata(Kind: "amdgpu.no.remote.memory",
11486 Node: llvm::MDNode::get(Context&: Builder.getContext(), MDs: {}));
11487 }
11488 Res.first = RMWInst;
11489 // not needed except in case of postfix captures. Generate anyway for
11490 // consistency with the else part. Will be removed with any DCE pass.
11491 // AtomicRMWInst::Xchg does not have a coressponding instruction.
11492 if (RMWOp == AtomicRMWInst::Xchg)
11493 Res.second = Res.first;
11494 else
11495 Res.second = emitRMWOpAsInstruction(Src1: Res.first, Src2: Expr, RMWOp);
11496 } else if (XElemTy->isStructTy()) {
11497 LoadInst *OldVal =
11498 Builder.CreateLoad(Ty: XElemTy, Ptr: X, Name: X->getName() + ".atomic.load");
11499 AtomicOrdering LoadAO = TransformReleaseAcquireRelease(AO);
11500 OldVal->setAtomic(Ordering: LoadAO);
11501 const DataLayout &LoadDL = OldVal->getModule()->getDataLayout();
11502 unsigned LoadSize = LoadDL.getTypeStoreSize(Ty: XElemTy);
11503
11504 OpenMPIRBuilder::AtomicInfo atomicInfo(
11505 &Builder, XElemTy, LoadSize * 8, LoadSize * 8, OldVal->getAlign(),
11506 OldVal->getAlign(), true /* UseLibcall */, AllocaIP, X);
11507 auto AtomicLoadRes = atomicInfo.EmitAtomicLoadLibcall(AO);
11508 BasicBlock *CurBB = Builder.GetInsertBlock();
11509 Instruction *CurBBTI = CurBB->getTerminatorOrNull();
11510 CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable();
11511 BasicBlock *ExitBB =
11512 CurBB->splitBasicBlock(I: CurBBTI, BBName: X->getName() + ".atomic.exit");
11513 BasicBlock *ContBB = CurBB->splitBasicBlock(I: CurBB->getTerminator(),
11514 BBName: X->getName() + ".atomic.cont");
11515 ContBB->getTerminator()->eraseFromParent();
11516 Builder.restoreIP(IP: AllocaIP);
11517 AllocaInst *NewAtomicAddr = Builder.CreateAlloca(Ty: XElemTy);
11518 NewAtomicAddr->setName(X->getName() + "x.new.val");
11519 Builder.SetInsertPoint(ContBB);
11520 llvm::PHINode *PHI = Builder.CreatePHI(Ty: OldVal->getType(), NumReservedValues: 2);
11521 PHI->addIncoming(V: AtomicLoadRes.first, BB: CurBB);
11522 Value *OldExprVal = PHI;
11523 Expected<Value *> CBResult = UpdateOp(OldExprVal, Builder);
11524 if (!CBResult)
11525 return CBResult.takeError();
11526 Value *Upd = *CBResult;
11527 Builder.CreateStore(Val: Upd, Ptr: NewAtomicAddr);
11528 AtomicOrdering Failure =
11529 llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: AO);
11530 auto Result = atomicInfo.EmitAtomicCompareExchangeLibcall(
11531 ExpectedVal: AtomicLoadRes.second, DesiredVal: NewAtomicAddr, Success: AO, Failure);
11532 LoadInst *PHILoad = Builder.CreateLoad(Ty: XElemTy, Ptr: Result.first);
11533 PHI->addIncoming(V: PHILoad, BB: Builder.GetInsertBlock());
11534 Builder.CreateCondBr(Cond: Result.second, True: ExitBB, False: ContBB);
11535 OldVal->eraseFromParent();
11536 Res.first = OldExprVal;
11537 Res.second = Upd;
11538
11539 if (UnreachableInst *ExitTI =
11540 dyn_cast<UnreachableInst>(Val: ExitBB->getTerminator())) {
11541 CurBBTI->eraseFromParent();
11542 Builder.SetInsertPoint(ExitBB);
11543 } else {
11544 Builder.SetInsertPoint(ExitTI);
11545 }
11546 } else {
11547 IntegerType *IntCastTy =
11548 IntegerType::get(C&: M.getContext(), NumBits: XElemTy->getScalarSizeInBits());
11549 LoadInst *OldVal =
11550 Builder.CreateLoad(Ty: IntCastTy, Ptr: X, Name: X->getName() + ".atomic.load");
11551 AtomicOrdering LoadAO = TransformReleaseAcquireRelease(AO);
11552 OldVal->setAtomic(Ordering: LoadAO);
11553 // CurBB
11554 // | /---\
11555 // ContBB |
11556 // | \---/
11557 // ExitBB
11558 BasicBlock *CurBB = Builder.GetInsertBlock();
11559 Instruction *CurBBTI = CurBB->getTerminatorOrNull();
11560 CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable();
11561 BasicBlock *ExitBB =
11562 CurBB->splitBasicBlock(I: CurBBTI, BBName: X->getName() + ".atomic.exit");
11563 BasicBlock *ContBB = CurBB->splitBasicBlock(I: CurBB->getTerminator(),
11564 BBName: X->getName() + ".atomic.cont");
11565 ContBB->getTerminator()->eraseFromParent();
11566 Builder.restoreIP(IP: AllocaIP);
11567 AllocaInst *NewAtomicAddr = Builder.CreateAlloca(Ty: XElemTy);
11568 NewAtomicAddr->setName(X->getName() + "x.new.val");
11569 Builder.SetInsertPoint(ContBB);
11570 llvm::PHINode *PHI = Builder.CreatePHI(Ty: OldVal->getType(), NumReservedValues: 2);
11571 PHI->addIncoming(V: OldVal, BB: CurBB);
11572 bool IsIntTy = XElemTy->isIntegerTy();
11573 Value *OldExprVal = PHI;
11574 if (!IsIntTy) {
11575 if (XElemTy->isFloatingPointTy()) {
11576 OldExprVal = Builder.CreateBitCast(V: PHI, DestTy: XElemTy,
11577 Name: X->getName() + ".atomic.fltCast");
11578 } else {
11579 OldExprVal = Builder.CreateIntToPtr(V: PHI, DestTy: XElemTy,
11580 Name: X->getName() + ".atomic.ptrCast");
11581 }
11582 }
11583
11584 Expected<Value *> CBResult = UpdateOp(OldExprVal, Builder);
11585 if (!CBResult)
11586 return CBResult.takeError();
11587 Value *Upd = *CBResult;
11588 Builder.CreateStore(Val: Upd, Ptr: NewAtomicAddr);
11589 LoadInst *DesiredVal = Builder.CreateLoad(Ty: IntCastTy, Ptr: NewAtomicAddr);
11590 AtomicOrdering Failure =
11591 llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: AO);
11592 AtomicCmpXchgInst *Result = Builder.CreateAtomicCmpXchg(
11593 Ptr: X, Cmp: PHI, New: DesiredVal, Align: llvm::MaybeAlign(), SuccessOrdering: AO, FailureOrdering: Failure);
11594 Result->setVolatile(VolatileX);
11595 Value *PreviousVal = Builder.CreateExtractValue(Agg: Result, /*Idxs=*/0);
11596 Value *SuccessFailureVal = Builder.CreateExtractValue(Agg: Result, /*Idxs=*/1);
11597 PHI->addIncoming(V: PreviousVal, BB: Builder.GetInsertBlock());
11598 Builder.CreateCondBr(Cond: SuccessFailureVal, True: ExitBB, False: ContBB);
11599
11600 Res.first = OldExprVal;
11601 Res.second = Upd;
11602
11603 // set Insertion point in exit block
11604 if (UnreachableInst *ExitTI =
11605 dyn_cast<UnreachableInst>(Val: ExitBB->getTerminator())) {
11606 CurBBTI->eraseFromParent();
11607 Builder.SetInsertPoint(ExitBB);
11608 } else {
11609 Builder.SetInsertPoint(ExitTI);
11610 }
11611 }
11612
11613 return Res;
11614}
11615
11616OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createAtomicCapture(
11617 const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X,
11618 AtomicOpValue &V, Value *Expr, AtomicOrdering AO,
11619 AtomicRMWInst::BinOp RMWOp, AtomicUpdateCallbackTy &UpdateOp,
11620 bool UpdateExpr, bool IsPostfixUpdate, bool IsXBinopExpr,
11621 bool IsIgnoreDenormalMode, bool IsFineGrainedMemory, bool IsRemoteMemory) {
11622 if (!updateToLocation(Loc))
11623 return Loc.IP;
11624
11625 LLVM_DEBUG({
11626 Type *XTy = X.Var->getType();
11627 assert(XTy->isPointerTy() &&
11628 "OMP Atomic expects a pointer to target memory");
11629 Type *XElemTy = X.ElemTy;
11630 assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||
11631 XElemTy->isPointerTy() || XElemTy->isStructTy()) &&
11632 "OMP atomic capture expected a scalar or struct type");
11633 assert((RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) &&
11634 "OpenMP atomic does not support LT or GT operations");
11635 });
11636
11637 // If UpdateExpr is 'x' updated with some `expr` not based on 'x',
11638 // 'x' is simply atomically rewritten with 'expr'.
11639 AtomicRMWInst::BinOp AtomicOp = (UpdateExpr ? RMWOp : AtomicRMWInst::Xchg);
11640 Expected<std::pair<Value *, Value *>> AtomicResult = emitAtomicUpdate(
11641 AllocaIP, X: X.Var, XElemTy: X.ElemTy, Expr, AO, RMWOp: AtomicOp, UpdateOp, VolatileX: X.IsVolatile,
11642 IsXBinopExpr, IsIgnoreDenormalMode, IsFineGrainedMemory, IsRemoteMemory);
11643 if (!AtomicResult)
11644 return AtomicResult.takeError();
11645 Value *CapturedVal =
11646 (IsPostfixUpdate ? AtomicResult->first : AtomicResult->second);
11647 Builder.CreateStore(Val: CapturedVal, Ptr: V.Var, isVolatile: V.IsVolatile);
11648
11649 checkAndEmitFlushAfterAtomic(Loc, AO, AK: AtomicKind::Capture);
11650 return Builder.saveIP();
11651}
11652
11653OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicCompare(
11654 const LocationDescription &Loc, AtomicOpValue &X, AtomicOpValue &V,
11655 AtomicOpValue &R, Value *E, Value *D, AtomicOrdering AO,
11656 omp::OMPAtomicCompareOp Op, bool IsXBinopExpr, bool IsPostfixUpdate,
11657 bool IsFailOnly, bool IsWeak) {
11658
11659 AtomicOrdering Failure = AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: AO);
11660 return createAtomicCompare(Loc, X, V, R, E, D, AO, Op, IsXBinopExpr,
11661 IsPostfixUpdate, IsFailOnly, Failure, IsWeak);
11662}
11663
11664OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicCompare(
11665 const LocationDescription &Loc, AtomicOpValue &X, AtomicOpValue &V,
11666 AtomicOpValue &R, Value *E, Value *D, AtomicOrdering AO,
11667 omp::OMPAtomicCompareOp Op, bool IsXBinopExpr, bool IsPostfixUpdate,
11668 bool IsFailOnly, AtomicOrdering Failure, bool IsWeak) {
11669
11670 if (!updateToLocation(Loc))
11671 return Loc.IP;
11672
11673 assert(X.Var->getType()->isPointerTy() &&
11674 "OMP atomic expects a pointer to target memory");
11675 // compare capture
11676 if (V.Var) {
11677 assert(V.Var->getType()->isPointerTy() && "v.var must be of pointer type");
11678 assert(V.ElemTy == X.ElemTy && "x and v must be of same type");
11679 }
11680
11681 bool IsInteger = E->getType()->isIntegerTy();
11682
11683 if (Op == OMPAtomicCompareOp::EQ) {
11684 // OldValue and SuccessOrFail are set below and used in the shared V.Var /
11685 // R.Var handling.
11686 Value *OldValue = nullptr;
11687 Value *SuccessOrFail = nullptr;
11688
11689 if (!IsInteger && HandleFPNegZero) {
11690 // IEEE 754 special cases for cmpxchg (which is bitwise):
11691 // 1. -0.0 == +0.0 but they have different bit patterns.
11692 // 2. NaN != NaN but identical NaN bit patterns would match.
11693 //
11694 // CurBB:
11695 // %e_int = bitcast E to intN
11696 // %d_int = bitcast D to intN
11697 // %x_curr = load atomic intN, X
11698 // %x_fp = bitcast %x_curr to FP
11699 // %e_is_nan = fcmp uno E, E
11700 // %x_is_nan = fcmp uno %x_fp, %x_fp
11701 // %either_nan = or %e_is_nan, %x_is_nan
11702 // br %either_nan, NaNBB, NotNaNBB
11703 // NaNBB: ; NaN == anything is always false
11704 // br ExitBB
11705 // NotNaNBB:
11706 // %x_is_zero = fcmp oeq %x_fp, 0.0
11707 // %e_is_zero = fcmp oeq E, 0.0
11708 // %both_zero = and %x_is_zero, %e_is_zero
11709 // br %both_zero, ZeroBB, NormalBB
11710 // ZeroBB: ; both ±0.0 → x = d
11711 // cmpxchg X, %x_curr, %d_int
11712 // br ExitBB
11713 // NormalBB: ; original path
11714 // cmpxchg X, %e_int, %d_int
11715 // br ExitBB
11716 // ExitBB:
11717 // phi merge
11718 IntegerType *IntCastTy =
11719 IntegerType::get(C&: M.getContext(), NumBits: X.ElemTy->getScalarSizeInBits());
11720 Value *EBCast = Builder.CreateBitCast(V: E, DestTy: IntCastTy);
11721 Value *DBCast = Builder.CreateBitCast(V: D, DestTy: IntCastTy);
11722
11723 // Load X atomically.
11724 LoadInst *XCurr = Builder.CreateLoad(Ty: IntCastTy, Ptr: X.Var,
11725 Name: X.Var->getName() + ".atomic.load");
11726 XCurr->setAtomic(Ordering: AtomicOrdering::Monotonic);
11727 Value *XFP = Builder.CreateBitCast(V: XCurr, DestTy: X.ElemTy);
11728
11729 // IEEE 754: NaN != NaN, but cmpxchg would succeed if E and X have
11730 // the same NaN bit pattern. Skip cmpxchg when either is NaN.
11731 Value *EIsNaN = Builder.CreateFCmpUNO(LHS: E, RHS: E, Name: "atomic.e.isnan");
11732 Value *XIsNaN = Builder.CreateFCmpUNO(LHS: XFP, RHS: XFP, Name: "atomic.x.isnan");
11733 Value *EitherNaN = Builder.CreateOr(LHS: EIsNaN, RHS: XIsNaN, Name: "atomic.either.nan");
11734
11735 BasicBlock *CurBB = Builder.GetInsertBlock();
11736 Function *F = CurBB->getParent();
11737 Instruction *CurBBTI = CurBB->getTerminatorOrNull();
11738 CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable();
11739 BasicBlock *ExitBB =
11740 CurBB->splitBasicBlock(I: CurBBTI, BBName: X.Var->getName() + ".atomic.exit");
11741 BasicBlock *NaNBB = BasicBlock::Create(
11742 Context&: M.getContext(), Name: X.Var->getName() + ".atomic.nan", Parent: F, InsertBefore: ExitBB);
11743 BasicBlock *NotNaNBB = BasicBlock::Create(
11744 Context&: M.getContext(), Name: X.Var->getName() + ".atomic.notnan", Parent: F, InsertBefore: ExitBB);
11745 BasicBlock *ZeroBB = BasicBlock::Create(
11746 Context&: M.getContext(), Name: X.Var->getName() + ".atomic.zero", Parent: F, InsertBefore: ExitBB);
11747 BasicBlock *NormalBB = BasicBlock::Create(
11748 Context&: M.getContext(), Name: X.Var->getName() + ".atomic.normal", Parent: F, InsertBefore: ExitBB);
11749
11750 // If either E or X is NaN → NaNBB (always fails), else check for ±0.0.
11751 CurBB->getTerminator()->eraseFromParent();
11752 Builder.SetInsertPoint(CurBB);
11753 Builder.CreateCondBr(Cond: EitherNaN, True: NaNBB, False: NotNaNBB);
11754
11755 // NaNBB: NaN == anything is always false; skip cmpxchg.
11756 Builder.SetInsertPoint(NaNBB);
11757 Builder.CreateBr(Dest: ExitBB);
11758
11759 // NotNaNBB: check both X and E for ±0.0.
11760 Builder.SetInsertPoint(NotNaNBB);
11761 Value *XIsZero =
11762 Builder.CreateFCmpOEQ(LHS: XFP, RHS: ConstantFP::getZero(Ty: X.ElemTy),
11763 Name: X.Var->getName() + ".atomic.xiszero");
11764 Value *EIsZero = Builder.CreateFCmpOEQ(LHS: E, RHS: ConstantFP::getZero(Ty: X.ElemTy),
11765 Name: "atomic.e.iszero");
11766 Value *BothZero = Builder.CreateAnd(LHS: XIsZero, RHS: EIsZero, Name: "atomic.both.zero");
11767 Builder.CreateCondBr(Cond: BothZero, True: ZeroBB, False: NormalBB);
11768
11769 // ZeroBB: cmpxchg with X's loaded bit-pattern.
11770 Builder.SetInsertPoint(ZeroBB);
11771 AtomicCmpXchgInst *ResZero = Builder.CreateAtomicCmpXchg(
11772 Ptr: X.Var, Cmp: XCurr, New: DBCast, Align: MaybeAlign(), SuccessOrdering: AO, FailureOrdering: Failure);
11773 ResZero->setWeak(IsWeak);
11774 Value *OldZero = Builder.CreateExtractValue(Agg: ResZero, /*Idxs=*/0);
11775 Value *OkZero = Builder.CreateExtractValue(Agg: ResZero, /*Idxs=*/1);
11776 Builder.CreateBr(Dest: ExitBB);
11777
11778 // NormalBB: original bitwise cmpxchg.
11779 Builder.SetInsertPoint(NormalBB);
11780 AtomicCmpXchgInst *ResNormal = Builder.CreateAtomicCmpXchg(
11781 Ptr: X.Var, Cmp: EBCast, New: DBCast, Align: MaybeAlign(), SuccessOrdering: AO, FailureOrdering: Failure);
11782 ResNormal->setWeak(IsWeak);
11783 Value *OldNormal = Builder.CreateExtractValue(Agg: ResNormal, /*Idxs=*/0);
11784 Value *OkNormal = Builder.CreateExtractValue(Agg: ResNormal, /*Idxs=*/1);
11785 Builder.CreateBr(Dest: ExitBB);
11786
11787 // ExitBB: merge results from NaN, Zero, and Normal paths.
11788 Builder.SetInsertPoint(TheBB: ExitBB, IP: ExitBB->begin());
11789 PHINode *OldIntPHI =
11790 Builder.CreatePHI(Ty: IntCastTy, NumReservedValues: 3, Name: X.Var->getName() + ".atomic.old");
11791 OldIntPHI->addIncoming(V: XCurr, BB: NaNBB);
11792 OldIntPHI->addIncoming(V: OldZero, BB: ZeroBB);
11793 OldIntPHI->addIncoming(V: OldNormal, BB: NormalBB);
11794 PHINode *SuccessPHI = Builder.CreatePHI(Ty: Builder.getInt1Ty(), NumReservedValues: 3,
11795 Name: X.Var->getName() + ".atomic.ok");
11796 SuccessPHI->addIncoming(V: Builder.getFalse(), BB: NaNBB);
11797 SuccessPHI->addIncoming(V: OkZero, BB: ZeroBB);
11798 SuccessPHI->addIncoming(V: OkNormal, BB: NormalBB);
11799
11800 if (isa<UnreachableInst>(Val: ExitBB->getTerminator())) {
11801 CurBBTI->eraseFromParent();
11802 Builder.SetInsertPoint(ExitBB);
11803 } else {
11804 Builder.SetInsertPoint(&*ExitBB->getFirstNonPHIIt());
11805 }
11806
11807 OldValue = Builder.CreateBitCast(V: OldIntPHI, DestTy: X.ElemTy,
11808 Name: X.Var->getName() + ".atomic.old.fp");
11809 SuccessOrFail = SuccessPHI;
11810 } else {
11811 AtomicCmpXchgInst *Result = nullptr;
11812 if (!IsInteger) {
11813 IntegerType *IntCastTy =
11814 IntegerType::get(C&: M.getContext(), NumBits: X.ElemTy->getScalarSizeInBits());
11815 Value *EBCast = Builder.CreateBitCast(V: E, DestTy: IntCastTy);
11816 Value *DBCast = Builder.CreateBitCast(V: D, DestTy: IntCastTy);
11817 Result = Builder.CreateAtomicCmpXchg(Ptr: X.Var, Cmp: EBCast, New: DBCast,
11818 Align: MaybeAlign(), SuccessOrdering: AO, FailureOrdering: Failure);
11819 } else {
11820 Result =
11821 Builder.CreateAtomicCmpXchg(Ptr: X.Var, Cmp: E, New: D, Align: MaybeAlign(), SuccessOrdering: AO, FailureOrdering: Failure);
11822 }
11823 Result->setWeak(IsWeak);
11824
11825 if (V.Var) {
11826 OldValue = Builder.CreateExtractValue(Agg: Result, /*Idxs=*/0);
11827 if (!IsInteger)
11828 OldValue = Builder.CreateBitCast(V: OldValue, DestTy: X.ElemTy);
11829 assert(OldValue->getType() == V.ElemTy &&
11830 "OldValue and V must be of same type");
11831 if (IsPostfixUpdate) {
11832 Builder.CreateStore(Val: OldValue, Ptr: V.Var, isVolatile: V.IsVolatile);
11833 } else {
11834 SuccessOrFail = Builder.CreateExtractValue(Agg: Result, /*Idxs=*/1);
11835 if (IsFailOnly) {
11836 BasicBlock *CurBB = Builder.GetInsertBlock();
11837 Instruction *CurBBTI = CurBB->getTerminatorOrNull();
11838 CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable();
11839 BasicBlock *ExitBB = CurBB->splitBasicBlock(
11840 I: CurBBTI, BBName: X.Var->getName() + ".atomic.exit");
11841 BasicBlock *ContBB = CurBB->splitBasicBlock(
11842 I: CurBB->getTerminator(), BBName: X.Var->getName() + ".atomic.cont");
11843 ContBB->getTerminator()->eraseFromParent();
11844 CurBB->getTerminator()->eraseFromParent();
11845
11846 Builder.CreateCondBr(Cond: SuccessOrFail, True: ExitBB, False: ContBB);
11847
11848 Builder.SetInsertPoint(ContBB);
11849 Builder.CreateStore(Val: OldValue, Ptr: V.Var);
11850 Builder.CreateBr(Dest: ExitBB);
11851
11852 if (UnreachableInst *ExitTI =
11853 dyn_cast<UnreachableInst>(Val: ExitBB->getTerminator())) {
11854 CurBBTI->eraseFromParent();
11855 Builder.SetInsertPoint(ExitBB);
11856 } else {
11857 Builder.SetInsertPoint(ExitTI);
11858 }
11859 } else {
11860 Value *CapturedValue =
11861 Builder.CreateSelect(C: SuccessOrFail, True: E, False: OldValue);
11862 Builder.CreateStore(Val: CapturedValue, Ptr: V.Var, isVolatile: V.IsVolatile);
11863 }
11864 }
11865 }
11866 // The comparison result has to be stored.
11867 if (R.Var) {
11868 assert(R.Var->getType()->isPointerTy() &&
11869 "r.var must be of pointer type");
11870 assert(R.ElemTy->isIntegerTy() && "r must be of integral type");
11871
11872 Value *SuccessFailureVal =
11873 Builder.CreateExtractValue(Agg: Result, /*Idxs=*/1);
11874 Value *ResultCast =
11875 R.IsSigned ? Builder.CreateSExt(V: SuccessFailureVal, DestTy: R.ElemTy)
11876 : Builder.CreateZExt(V: SuccessFailureVal, DestTy: R.ElemTy);
11877 Builder.CreateStore(Val: ResultCast, Ptr: R.Var, isVolatile: R.IsVolatile);
11878 }
11879 }
11880
11881 // For the HandleFPNegZero path, handle V.Var and R.Var using the
11882 // pre-computed OldValue and SuccessOrFail.
11883 if (HandleFPNegZero && !IsInteger) {
11884 if (V.Var) {
11885 assert(OldValue->getType() == V.ElemTy &&
11886 "OldValue and V must be of same type");
11887 if (IsPostfixUpdate) {
11888 Builder.CreateStore(Val: OldValue, Ptr: V.Var, isVolatile: V.IsVolatile);
11889 } else {
11890 if (IsFailOnly) {
11891 BasicBlock *CurBB = Builder.GetInsertBlock();
11892 Instruction *CurBBTI = CurBB->getTerminatorOrNull();
11893 CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable();
11894 BasicBlock *ExitBB = CurBB->splitBasicBlock(
11895 I: CurBBTI, BBName: X.Var->getName() + ".atomic.exit");
11896 BasicBlock *ContBB = CurBB->splitBasicBlock(
11897 I: CurBB->getTerminator(), BBName: X.Var->getName() + ".atomic.cont");
11898 ContBB->getTerminator()->eraseFromParent();
11899 CurBB->getTerminator()->eraseFromParent();
11900
11901 Builder.CreateCondBr(Cond: SuccessOrFail, True: ExitBB, False: ContBB);
11902
11903 Builder.SetInsertPoint(ContBB);
11904 Builder.CreateStore(Val: OldValue, Ptr: V.Var);
11905 Builder.CreateBr(Dest: ExitBB);
11906
11907 if (UnreachableInst *ExitTI =
11908 dyn_cast<UnreachableInst>(Val: ExitBB->getTerminator())) {
11909 CurBBTI->eraseFromParent();
11910 Builder.SetInsertPoint(ExitBB);
11911 } else {
11912 Builder.SetInsertPoint(ExitTI);
11913 }
11914 } else {
11915 Value *CapturedValue =
11916 Builder.CreateSelect(C: SuccessOrFail, True: E, False: OldValue);
11917 Builder.CreateStore(Val: CapturedValue, Ptr: V.Var, isVolatile: V.IsVolatile);
11918 }
11919 }
11920 }
11921 // The comparison result has to be stored.
11922 if (R.Var) {
11923 assert(R.Var->getType()->isPointerTy() &&
11924 "r.var must be of pointer type");
11925 assert(R.ElemTy->isIntegerTy() && "r must be of integral type");
11926
11927 Value *ResultCast = R.IsSigned
11928 ? Builder.CreateSExt(V: SuccessOrFail, DestTy: R.ElemTy)
11929 : Builder.CreateZExt(V: SuccessOrFail, DestTy: R.ElemTy);
11930 Builder.CreateStore(Val: ResultCast, Ptr: R.Var, isVolatile: R.IsVolatile);
11931 }
11932 }
11933 } else {
11934 assert((Op == OMPAtomicCompareOp::MAX || Op == OMPAtomicCompareOp::MIN) &&
11935 "Op should be either max or min at this point");
11936 assert(!IsFailOnly && "IsFailOnly is only valid when the comparison is ==");
11937
11938 // Reverse the ordop as the OpenMP forms are different from LLVM forms.
11939 // Let's take max as example.
11940 // OpenMP form:
11941 // x = x > expr ? expr : x;
11942 // LLVM form:
11943 // *ptr = *ptr > val ? *ptr : val;
11944 // We need to transform to LLVM form.
11945 // x = x <= expr ? x : expr;
11946 AtomicRMWInst::BinOp NewOp;
11947 if (IsXBinopExpr) {
11948 if (IsInteger) {
11949 if (X.IsSigned)
11950 NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::Min
11951 : AtomicRMWInst::Max;
11952 else
11953 NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::UMin
11954 : AtomicRMWInst::UMax;
11955 } else {
11956 NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::FMin
11957 : AtomicRMWInst::FMax;
11958 }
11959 } else {
11960 if (IsInteger) {
11961 if (X.IsSigned)
11962 NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::Max
11963 : AtomicRMWInst::Min;
11964 else
11965 NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::UMax
11966 : AtomicRMWInst::UMin;
11967 } else {
11968 NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::FMax
11969 : AtomicRMWInst::FMin;
11970 }
11971 }
11972
11973 AtomicRMWInst *OldValue =
11974 Builder.CreateAtomicRMW(Op: NewOp, Ptr: X.Var, Val: E, Align: MaybeAlign(), Ordering: AO);
11975 if (V.Var) {
11976 Value *CapturedValue = nullptr;
11977 if (IsPostfixUpdate) {
11978 CapturedValue = OldValue;
11979 } else {
11980 CmpInst::Predicate Pred;
11981 switch (NewOp) {
11982 case AtomicRMWInst::Max:
11983 Pred = CmpInst::ICMP_SGT;
11984 break;
11985 case AtomicRMWInst::UMax:
11986 Pred = CmpInst::ICMP_UGT;
11987 break;
11988 case AtomicRMWInst::FMax:
11989 Pred = CmpInst::FCMP_OGT;
11990 break;
11991 case AtomicRMWInst::Min:
11992 Pred = CmpInst::ICMP_SLT;
11993 break;
11994 case AtomicRMWInst::UMin:
11995 Pred = CmpInst::ICMP_ULT;
11996 break;
11997 case AtomicRMWInst::FMin:
11998 Pred = CmpInst::FCMP_OLT;
11999 break;
12000 default:
12001 llvm_unreachable("unexpected comparison op");
12002 }
12003 Value *NonAtomicCmp = Builder.CreateCmp(Pred, LHS: OldValue, RHS: E);
12004 CapturedValue = Builder.CreateSelect(C: NonAtomicCmp, True: E, False: OldValue);
12005 }
12006 Builder.CreateStore(Val: CapturedValue, Ptr: V.Var, isVolatile: V.IsVolatile);
12007 }
12008 }
12009
12010 checkAndEmitFlushAfterAtomic(Loc, AO, AK: AtomicKind::Compare);
12011
12012 return Builder.saveIP();
12013}
12014
12015OpenMPIRBuilder::InsertPointOrErrorTy
12016OpenMPIRBuilder::createTeams(const LocationDescription &Loc,
12017 BodyGenCallbackTy BodyGenCB, Value *NumTeamsLower,
12018 Value *NumTeamsUpper, Value *ThreadLimit,
12019 Value *IfExpr) {
12020 if (!updateToLocation(Loc))
12021 return InsertPointTy();
12022
12023 uint32_t SrcLocStrSize;
12024 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
12025 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
12026 Function *CurrentFunction = Builder.GetInsertBlock()->getParent();
12027
12028 // Outer allocation basicblock is the entry block of the current function.
12029 BasicBlock &OuterAllocaBB = CurrentFunction->getEntryBlock();
12030 if (&OuterAllocaBB == Builder.GetInsertBlock()) {
12031 BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, Name: "teams.entry");
12032 Builder.SetInsertPoint(TheBB: BodyBB, IP: BodyBB->begin());
12033 }
12034
12035 // The current basic block is split into four basic blocks. After outlining,
12036 // they will be mapped as follows:
12037 // ```
12038 // def current_fn() {
12039 // current_basic_block:
12040 // br label %teams.exit
12041 // teams.exit:
12042 // ; instructions after teams
12043 // }
12044 //
12045 // def outlined_fn() {
12046 // teams.alloca:
12047 // br label %teams.body
12048 // teams.body:
12049 // ; instructions within teams body
12050 // }
12051 // ```
12052 BasicBlock *ExitBB = splitBB(Builder, /*CreateBranch=*/true, Name: "teams.exit");
12053 BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, Name: "teams.body");
12054 BasicBlock *AllocaBB =
12055 splitBB(Builder, /*CreateBranch=*/true, Name: "teams.alloca");
12056
12057 bool SubClausesPresent =
12058 (NumTeamsLower || NumTeamsUpper || ThreadLimit || IfExpr);
12059 // Push num_teams
12060 if (!Config.isTargetDevice() && SubClausesPresent) {
12061 assert((NumTeamsLower == nullptr || NumTeamsUpper != nullptr) &&
12062 "if lowerbound is non-null, then upperbound must also be non-null "
12063 "for bounds on num_teams");
12064
12065 if (NumTeamsUpper == nullptr)
12066 NumTeamsUpper = Builder.getInt32(C: 0);
12067
12068 if (NumTeamsLower == nullptr)
12069 NumTeamsLower = NumTeamsUpper;
12070
12071 if (IfExpr) {
12072 assert(IfExpr->getType()->isIntegerTy() &&
12073 "argument to if clause must be an integer value");
12074
12075 // upper = ifexpr ? upper : 1
12076 if (IfExpr->getType() != Int1)
12077 IfExpr = Builder.CreateICmpNE(LHS: IfExpr,
12078 RHS: ConstantInt::get(Ty: IfExpr->getType(), V: 0));
12079 NumTeamsUpper = Builder.CreateSelect(
12080 C: IfExpr, True: NumTeamsUpper, False: Builder.getInt32(C: 1), Name: "numTeamsUpper");
12081
12082 // lower = ifexpr ? lower : 1
12083 NumTeamsLower = Builder.CreateSelect(
12084 C: IfExpr, True: NumTeamsLower, False: Builder.getInt32(C: 1), Name: "numTeamsLower");
12085 }
12086
12087 if (ThreadLimit == nullptr)
12088 ThreadLimit = Builder.getInt32(C: 0);
12089
12090 // The __kmpc_push_num_teams_51 function expects int32 as the arguments. So,
12091 // truncate or sign extend the passed values to match the int32 parameters.
12092 Value *NumTeamsLowerInt32 =
12093 Builder.CreateSExtOrTrunc(V: NumTeamsLower, DestTy: Builder.getInt32Ty());
12094 Value *NumTeamsUpperInt32 =
12095 Builder.CreateSExtOrTrunc(V: NumTeamsUpper, DestTy: Builder.getInt32Ty());
12096 Value *ThreadLimitInt32 =
12097 Builder.CreateSExtOrTrunc(V: ThreadLimit, DestTy: Builder.getInt32Ty());
12098
12099 Value *ThreadNum = getOrCreateThreadID(Ident);
12100
12101 createRuntimeFunctionCall(
12102 Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_push_num_teams_51),
12103 Args: {Ident, ThreadNum, NumTeamsLowerInt32, NumTeamsUpperInt32,
12104 ThreadLimitInt32});
12105 }
12106 // Generate the body of teams.
12107 InsertPointTy AllocaIP(AllocaBB, AllocaBB->begin());
12108 InsertPointTy CodeGenIP(BodyBB, BodyBB->begin());
12109 if (Error Err = BodyGenCB(AllocaIP, CodeGenIP, ExitBB))
12110 return Err;
12111
12112 auto OI = std::make_unique<OutlineInfo>();
12113 OI->EntryBB = AllocaBB;
12114 OI->ExitBB = ExitBB;
12115 OI->OuterAllocBB = &OuterAllocaBB;
12116
12117 // Insert fake values for global tid and bound tid.
12118 SmallVector<Instruction *, 8> ToBeDeleted;
12119 InsertPointTy OuterAllocaIP(&OuterAllocaBB, OuterAllocaBB.begin());
12120 OI->ExcludeArgsFromAggregate.push_back(Elt: createFakeIntVal(
12121 Builder, OuterAllocaIP, ToBeDeleted, InnerAllocaIP: AllocaIP, Name: "gid", AsPtr: true));
12122 OI->ExcludeArgsFromAggregate.push_back(Elt: createFakeIntVal(
12123 Builder, OuterAllocaIP, ToBeDeleted, InnerAllocaIP: AllocaIP, Name: "tid", AsPtr: true));
12124
12125 auto HostPostOutlineCB = [this, Ident,
12126 ToBeDeleted](Function &OutlinedFn) mutable {
12127 // The stale call instruction will be replaced with a new call instruction
12128 // for runtime call with the outlined function.
12129
12130 assert(OutlinedFn.hasOneUse() &&
12131 "there must be a single user for the outlined function");
12132 CallInst *StaleCI = cast<CallInst>(Val: OutlinedFn.user_back());
12133 ToBeDeleted.push_back(Elt: StaleCI);
12134
12135 assert((OutlinedFn.arg_size() == 2 || OutlinedFn.arg_size() == 3) &&
12136 "Outlined function must have two or three arguments only");
12137
12138 bool HasShared = OutlinedFn.arg_size() == 3;
12139
12140 OutlinedFn.getArg(i: 0)->setName("global.tid.ptr");
12141 OutlinedFn.getArg(i: 1)->setName("bound.tid.ptr");
12142 if (HasShared)
12143 OutlinedFn.getArg(i: 2)->setName("data");
12144
12145 // Call to the runtime function for teams in the current function.
12146 assert(StaleCI && "Error while outlining - no CallInst user found for the "
12147 "outlined function.");
12148 Builder.SetInsertPoint(StaleCI);
12149 SmallVector<Value *> Args = {
12150 Ident, Builder.getInt32(C: StaleCI->arg_size() - 2), &OutlinedFn};
12151 if (HasShared)
12152 Args.push_back(Elt: StaleCI->getArgOperand(i: 2));
12153 createRuntimeFunctionCall(
12154 Callee: getOrCreateRuntimeFunctionPtr(
12155 FnID: omp::RuntimeFunction::OMPRTL___kmpc_fork_teams),
12156 Args);
12157
12158 Builder.ClearInsertionPoint();
12159 for (Instruction *I : llvm::reverse(C&: ToBeDeleted))
12160 I->eraseFromParent();
12161 };
12162
12163 if (!Config.isTargetDevice())
12164 OI->PostOutlineCB = HostPostOutlineCB;
12165
12166 addOutlineInfo(OI: std::move(OI));
12167
12168 Builder.SetInsertPoint(ExitBB);
12169
12170 return Builder.saveIP();
12171}
12172
12173OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createDistribute(
12174 const LocationDescription &Loc, InsertPointTy OuterAllocIP,
12175 ArrayRef<BasicBlock *> OuterDeallocBlocks, BodyGenCallbackTy BodyGenCB) {
12176 if (!updateToLocation(Loc))
12177 return InsertPointTy();
12178
12179 BasicBlock *OuterAllocaBB = OuterAllocIP.getBlock();
12180
12181 if (OuterAllocaBB == Builder.GetInsertBlock()) {
12182 BasicBlock *BodyBB =
12183 splitBB(Builder, /*CreateBranch=*/true, Name: "distribute.entry");
12184 Builder.SetInsertPoint(TheBB: BodyBB, IP: BodyBB->begin());
12185 }
12186 BasicBlock *ExitBB =
12187 splitBB(Builder, /*CreateBranch=*/true, Name: "distribute.exit");
12188 BasicBlock *BodyBB =
12189 splitBB(Builder, /*CreateBranch=*/true, Name: "distribute.body");
12190 BasicBlock *AllocaBB =
12191 splitBB(Builder, /*CreateBranch=*/true, Name: "distribute.alloca");
12192
12193 // Generate the body of distribute clause
12194 InsertPointTy AllocaIP(AllocaBB, AllocaBB->begin());
12195 InsertPointTy CodeGenIP(BodyBB, BodyBB->begin());
12196 if (Error Err = BodyGenCB(AllocaIP, CodeGenIP, ExitBB))
12197 return Err;
12198
12199 // When using target we use different runtime functions which require a
12200 // callback.
12201 if (Config.isTargetDevice()) {
12202 auto OI = std::make_unique<OutlineInfo>();
12203 OI->OuterAllocBB = OuterAllocIP.getBlock();
12204 OI->EntryBB = AllocaBB;
12205 OI->ExitBB = ExitBB;
12206 OI->OuterDeallocBBs.reserve(N: OuterDeallocBlocks.size());
12207 copy(Range&: OuterDeallocBlocks, Out: OI->OuterDeallocBBs.end());
12208
12209 addOutlineInfo(OI: std::move(OI));
12210 }
12211 Builder.SetInsertPoint(ExitBB);
12212
12213 return Builder.saveIP();
12214}
12215
12216GlobalVariable *
12217OpenMPIRBuilder::createOffloadMapnames(SmallVectorImpl<llvm::Constant *> &Names,
12218 std::string VarName) {
12219 llvm::Constant *MapNamesArrayInit = llvm::ConstantArray::get(
12220 T: llvm::ArrayType::get(ElementType: llvm::PointerType::getUnqual(C&: M.getContext()),
12221 NumElements: Names.size()),
12222 V: Names);
12223 auto *MapNamesArrayGlobal = new llvm::GlobalVariable(
12224 M, MapNamesArrayInit->getType(),
12225 /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage, MapNamesArrayInit,
12226 VarName);
12227 return MapNamesArrayGlobal;
12228}
12229
12230// Create all simple and struct types exposed by the runtime and remember
12231// the llvm::PointerTypes of them for easy access later.
12232void OpenMPIRBuilder::initializeTypes(Module &M) {
12233 LLVMContext &Ctx = M.getContext();
12234 StructType *T;
12235 unsigned DefaultTargetAS = Config.getDefaultTargetAS();
12236 unsigned ProgramAS = M.getDataLayout().getProgramAddressSpace();
12237#define OMP_TYPE(VarName, InitValue) VarName = InitValue;
12238#define OMP_ARRAY_TYPE(VarName, ElemTy, ArraySize) \
12239 VarName##Ty = ArrayType::get(ElemTy, ArraySize); \
12240 VarName##PtrTy = PointerType::get(Ctx, DefaultTargetAS);
12241#define OMP_FUNCTION_TYPE(VarName, IsVarArg, ReturnType, ...) \
12242 VarName = FunctionType::get(ReturnType, {__VA_ARGS__}, IsVarArg); \
12243 VarName##Ptr = PointerType::get(Ctx, ProgramAS);
12244#define OMP_STRUCT_TYPE(VarName, StructName, Packed, ...) \
12245 T = StructType::getTypeByName(Ctx, StructName); \
12246 if (!T) \
12247 T = StructType::create(Ctx, {__VA_ARGS__}, StructName, Packed); \
12248 VarName = T; \
12249 VarName##Ptr = PointerType::get(Ctx, DefaultTargetAS);
12250#include "llvm/Frontend/OpenMP/OMPKinds.def"
12251}
12252
12253void OpenMPIRBuilder::OutlineInfo::collectBlocks(
12254 SmallPtrSetImpl<BasicBlock *> &BlockSet,
12255 SmallVectorImpl<BasicBlock *> &BlockVector) {
12256 SmallVector<BasicBlock *, 32> Worklist;
12257 BlockSet.insert(Ptr: EntryBB);
12258 BlockSet.insert(Ptr: ExitBB);
12259
12260 Worklist.push_back(Elt: EntryBB);
12261 while (!Worklist.empty()) {
12262 BasicBlock *BB = Worklist.pop_back_val();
12263 BlockVector.push_back(Elt: BB);
12264 for (BasicBlock *SuccBB : successors(BB))
12265 if (BlockSet.insert(Ptr: SuccBB).second)
12266 Worklist.push_back(Elt: SuccBB);
12267 }
12268}
12269
12270std::unique_ptr<CodeExtractor>
12271OpenMPIRBuilder::OutlineInfo::createCodeExtractor(ArrayRef<BasicBlock *> Blocks,
12272 bool ArgsInZeroAddressSpace,
12273 Twine Suffix) {
12274 return std::make_unique<CodeExtractor>(
12275 args&: Blocks, /* DominatorTree */ args: nullptr,
12276 /* AggregateArgs */ args: true,
12277 /* BlockFrequencyInfo */ args: nullptr,
12278 /* BranchProbabilityInfo */ args: nullptr,
12279 /* AssumptionCache */ args: nullptr,
12280 /* AllowVarArgs */ args: true,
12281 /* AllowAlloca */ args: true,
12282 /* AllocationBlock*/ args&: OuterAllocBB,
12283 /* DeallocationBlocks */ args: ArrayRef<BasicBlock *>(),
12284 /* Suffix */ args: Suffix.str(), args&: ArgsInZeroAddressSpace);
12285}
12286
12287std::unique_ptr<CodeExtractor> DeviceSharedMemOutlineInfo::createCodeExtractor(
12288 ArrayRef<BasicBlock *> Blocks, bool ArgsInZeroAddressSpace, Twine Suffix) {
12289 return std::make_unique<DeviceSharedMemCodeExtractor>(
12290 args&: OMPBuilder, args&: Blocks, /* DominatorTree */ args: nullptr,
12291 /* AggregateArgs */ args: true,
12292 /* BlockFrequencyInfo */ args: nullptr,
12293 /* BranchProbabilityInfo */ args: nullptr,
12294 /* AssumptionCache */ args: nullptr,
12295 /* AllowVarArgs */ args: true,
12296 /* AllowAlloca */ args: true,
12297 /* AllocationBlock*/ args&: OuterAllocBB,
12298 /* DeallocationBlocks */ args: OuterDeallocBBs.empty()
12299 ? SmallVector<BasicBlock *>{ExitBB}
12300 : OuterDeallocBBs,
12301 /* Suffix */ args: Suffix.str(), args&: ArgsInZeroAddressSpace);
12302}
12303
12304void OpenMPIRBuilder::createOffloadEntry(Constant *ID, Constant *Addr,
12305 uint64_t Size, int32_t Flags,
12306 GlobalValue::LinkageTypes,
12307 StringRef Name) {
12308 if (!Config.isGPU()) {
12309 llvm::offloading::emitOffloadingEntry(
12310 M, Kind: object::OffloadKind::OFK_OpenMP, Addr: ID,
12311 Name: Name.empty() ? Addr->getName() : Name, Size, Flags, /*Data=*/0);
12312 return;
12313 }
12314 // TODO: Add support for global variables on the device after declare target
12315 // support.
12316 Function *Fn = dyn_cast<Function>(Val: Addr);
12317 if (!Fn)
12318 return;
12319
12320 // Add a function attribute for the kernel.
12321 Fn->addFnAttr(Kind: "kernel");
12322 if (T.isAMDGCN())
12323 Fn->addFnAttr(Kind: "uniform-work-group-size");
12324 Fn->addFnAttr(Kind: Attribute::MustProgress);
12325}
12326
12327// We only generate metadata for function that contain target regions.
12328void OpenMPIRBuilder::createOffloadEntriesAndInfoMetadata(
12329 EmitMetadataErrorReportFunctionTy &ErrorFn) {
12330
12331 // If there are no entries, we don't need to do anything.
12332 if (OffloadInfoManager.empty())
12333 return;
12334
12335 LLVMContext &C = M.getContext();
12336 SmallVector<std::pair<const OffloadEntriesInfoManager::OffloadEntryInfo *,
12337 TargetRegionEntryInfo>,
12338 16>
12339 OrderedEntries(OffloadInfoManager.size());
12340
12341 // Auxiliary methods to create metadata values and strings.
12342 auto &&GetMDInt = [this](unsigned V) {
12343 return ConstantAsMetadata::get(C: ConstantInt::get(Ty: Builder.getInt32Ty(), V));
12344 };
12345
12346 auto &&GetMDString = [&C](StringRef V) { return MDString::get(Context&: C, Str: V); };
12347
12348 // Create the offloading info metadata node.
12349 NamedMDNode *MD = M.getOrInsertNamedMetadata(Name: "omp_offload.info");
12350 auto &&TargetRegionMetadataEmitter =
12351 [&C, MD, &OrderedEntries, &GetMDInt, &GetMDString](
12352 const TargetRegionEntryInfo &EntryInfo,
12353 const OffloadEntriesInfoManager::OffloadEntryInfoTargetRegion &E) {
12354 // Generate metadata for target regions. Each entry of this metadata
12355 // contains:
12356 // - Entry 0 -> Kind of this type of metadata (0).
12357 // - Entry 1 -> Device ID of the file where the entry was identified.
12358 // - Entry 2 -> File ID of the file where the entry was identified.
12359 // - Entry 3 -> Mangled name of the function where the entry was
12360 // identified.
12361 // - Entry 4 -> Line in the file where the entry was identified.
12362 // - Entry 5 -> Count of regions at this DeviceID/FilesID/Line.
12363 // - Entry 6 -> Order the entry was created.
12364 // The first element of the metadata node is the kind.
12365 Metadata *Ops[] = {
12366 GetMDInt(E.getKind()), GetMDInt(EntryInfo.DeviceID),
12367 GetMDInt(EntryInfo.FileID), GetMDString(EntryInfo.ParentName),
12368 GetMDInt(EntryInfo.Line), GetMDInt(EntryInfo.Count),
12369 GetMDInt(E.getOrder())};
12370
12371 // Save this entry in the right position of the ordered entries array.
12372 OrderedEntries[E.getOrder()] = std::make_pair(x: &E, y: EntryInfo);
12373
12374 // Add metadata to the named metadata node.
12375 MD->addOperand(M: MDNode::get(Context&: C, MDs: Ops));
12376 };
12377
12378 OffloadInfoManager.actOnTargetRegionEntriesInfo(Action: TargetRegionMetadataEmitter);
12379
12380 // Create function that emits metadata for each device global variable entry;
12381 auto &&DeviceGlobalVarMetadataEmitter =
12382 [&C, &OrderedEntries, &GetMDInt, &GetMDString, MD](
12383 StringRef MangledName,
12384 const OffloadEntriesInfoManager::OffloadEntryInfoDeviceGlobalVar &E) {
12385 // Generate metadata for global variables. Each entry of this metadata
12386 // contains:
12387 // - Entry 0 -> Kind of this type of metadata (1).
12388 // - Entry 1 -> Mangled name of the variable.
12389 // - Entry 2 -> Declare target kind.
12390 // - Entry 3 -> Order the entry was created.
12391 // The first element of the metadata node is the kind.
12392 Metadata *Ops[] = {GetMDInt(E.getKind()), GetMDString(MangledName),
12393 GetMDInt(E.getFlags()), GetMDInt(E.getOrder())};
12394
12395 // Save this entry in the right position of the ordered entries array.
12396 TargetRegionEntryInfo varInfo(MangledName, 0, 0, 0);
12397 OrderedEntries[E.getOrder()] = std::make_pair(x: &E, y&: varInfo);
12398
12399 // Add metadata to the named metadata node.
12400 MD->addOperand(M: MDNode::get(Context&: C, MDs: Ops));
12401 };
12402
12403 OffloadInfoManager.actOnDeviceGlobalVarEntriesInfo(
12404 Action: DeviceGlobalVarMetadataEmitter);
12405
12406 for (const auto &E : OrderedEntries) {
12407 assert(E.first && "All ordered entries must exist!");
12408 if (const auto *CE =
12409 dyn_cast<OffloadEntriesInfoManager::OffloadEntryInfoTargetRegion>(
12410 Val: E.first)) {
12411 if (!CE->getID() || !CE->getAddress()) {
12412 // Do not blame the entry if the parent funtion is not emitted.
12413 TargetRegionEntryInfo EntryInfo = E.second;
12414 StringRef FnName = EntryInfo.ParentName;
12415 if (!M.getNamedValue(Name: FnName))
12416 continue;
12417 ErrorFn(EMIT_MD_TARGET_REGION_ERROR, EntryInfo);
12418 continue;
12419 }
12420 createOffloadEntry(ID: CE->getID(), Addr: CE->getAddress(),
12421 /*Size=*/0, Flags: CE->getFlags(),
12422 GlobalValue::WeakAnyLinkage);
12423 } else if (const auto *CE = dyn_cast<
12424 OffloadEntriesInfoManager::OffloadEntryInfoDeviceGlobalVar>(
12425 Val: E.first)) {
12426 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind Flags =
12427 static_cast<OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind>(
12428 CE->getFlags());
12429 switch (Flags) {
12430 case OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter:
12431 case OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo:
12432 if (Config.isTargetDevice() && Config.hasRequiresUnifiedSharedMemory())
12433 continue;
12434 if (!CE->getAddress()) {
12435 ErrorFn(EMIT_MD_DECLARE_TARGET_ERROR, E.second);
12436 continue;
12437 }
12438 // The vaiable has no definition - no need to add the entry.
12439 if (CE->getVarSize() == 0)
12440 continue;
12441 break;
12442 case OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink:
12443 assert(((Config.isTargetDevice() && !CE->getAddress()) ||
12444 (!Config.isTargetDevice() && CE->getAddress())) &&
12445 "Declaret target link address is set.");
12446 if (Config.isTargetDevice())
12447 continue;
12448 if (!CE->getAddress()) {
12449 ErrorFn(EMIT_MD_GLOBAL_VAR_LINK_ERROR, TargetRegionEntryInfo());
12450 continue;
12451 }
12452 break;
12453 case OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect:
12454 case OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirectVTable:
12455 if (!CE->getAddress()) {
12456 ErrorFn(EMIT_MD_GLOBAL_VAR_INDIRECT_ERROR, E.second);
12457 continue;
12458 }
12459 break;
12460 default:
12461 break;
12462 }
12463
12464 // Hidden or internal symbols on the device are not externally visible.
12465 // We should not attempt to register them by creating an offloading
12466 // entry. Indirect variables are handled separately on the device.
12467 if (auto *GV = dyn_cast<GlobalValue>(Val: CE->getAddress()))
12468 if ((GV->hasLocalLinkage() || GV->hasHiddenVisibility()) &&
12469 (Flags !=
12470 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect &&
12471 Flags != OffloadEntriesInfoManager::
12472 OMPTargetGlobalVarEntryIndirectVTable))
12473 continue;
12474
12475 // Indirect globals need to use a special name that doesn't match the name
12476 // of the associated host global.
12477 if (Flags == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect ||
12478 Flags ==
12479 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirectVTable)
12480 createOffloadEntry(ID: CE->getAddress(), Addr: CE->getAddress(), Size: CE->getVarSize(),
12481 Flags, CE->getLinkage(), Name: CE->getVarName());
12482 else
12483 createOffloadEntry(ID: CE->getAddress(), Addr: CE->getAddress(), Size: CE->getVarSize(),
12484 Flags, CE->getLinkage());
12485
12486 } else {
12487 llvm_unreachable("Unsupported entry kind.");
12488 }
12489 }
12490
12491 // Emit requires directive globals to a special entry so the runtime can
12492 // register them when the device image is loaded.
12493 // TODO: This reduces the offloading entries to a 32-bit integer. Offloading
12494 // entries should be redesigned to better suit this use-case.
12495 if (Config.hasRequiresFlags() && !Config.isTargetDevice())
12496 offloading::emitOffloadingEntry(
12497 M, Kind: object::OffloadKind::OFK_OpenMP,
12498 Addr: Constant::getNullValue(Ty: PointerType::getUnqual(C&: M.getContext())),
12499 Name: ".requires", /*Size=*/0,
12500 Flags: OffloadEntriesInfoManager::OMPTargetGlobalRegisterRequires,
12501 Data: Config.getRequiresFlags());
12502}
12503
12504void TargetRegionEntryInfo::getTargetRegionEntryFnName(
12505 SmallVectorImpl<char> &Name, StringRef ParentName, unsigned DeviceID,
12506 unsigned FileID, unsigned Line, unsigned Count) {
12507 raw_svector_ostream OS(Name);
12508 OS << KernelNamePrefix << llvm::format(Fmt: "%x", Vals: DeviceID)
12509 << llvm::format(Fmt: "_%x_", Vals: FileID) << ParentName << "_l" << Line;
12510 if (Count)
12511 OS << "_" << Count;
12512}
12513
12514void OffloadEntriesInfoManager::getTargetRegionEntryFnName(
12515 SmallVectorImpl<char> &Name, const TargetRegionEntryInfo &EntryInfo) {
12516 unsigned NewCount = getTargetRegionEntryInfoCount(EntryInfo);
12517 TargetRegionEntryInfo::getTargetRegionEntryFnName(
12518 Name, ParentName: EntryInfo.ParentName, DeviceID: EntryInfo.DeviceID, FileID: EntryInfo.FileID,
12519 Line: EntryInfo.Line, Count: NewCount);
12520}
12521
12522TargetRegionEntryInfo
12523OpenMPIRBuilder::getTargetEntryUniqueInfo(FileIdentifierInfoCallbackTy CallBack,
12524 vfs::FileSystem &VFS,
12525 StringRef ParentName) {
12526 sys::fs::UniqueID ID(0xdeadf17e, 0);
12527 auto FileIDInfo = CallBack();
12528 uint64_t FileID = 0;
12529 if (ErrorOr<vfs::Status> Status = VFS.status(Path: std::get<0>(t&: FileIDInfo))) {
12530 ID = Status->getUniqueID();
12531 FileID = Status->getUniqueID().getFile();
12532 } else {
12533 // If the inode ID could not be determined, create a hash value
12534 // the current file name and use that as an ID.
12535 FileID = hash_value(arg: std::get<0>(t&: FileIDInfo));
12536 }
12537
12538 return TargetRegionEntryInfo(ParentName, ID.getDevice(), FileID,
12539 std::get<1>(t&: FileIDInfo));
12540}
12541
12542unsigned OpenMPIRBuilder::getFlagMemberOffset() {
12543 unsigned Offset = 0;
12544 for (uint64_t Remain =
12545 static_cast<std::underlying_type_t<omp::OpenMPOffloadMappingFlags>>(
12546 omp::OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);
12547 !(Remain & 1); Remain = Remain >> 1)
12548 Offset++;
12549 return Offset;
12550}
12551
12552omp::OpenMPOffloadMappingFlags
12553OpenMPIRBuilder::getMemberOfFlag(unsigned Position) {
12554 // Rotate by getFlagMemberOffset() bits.
12555 return static_cast<omp::OpenMPOffloadMappingFlags>(((uint64_t)Position + 1)
12556 << getFlagMemberOffset());
12557}
12558
12559void OpenMPIRBuilder::setCorrectMemberOfFlag(
12560 omp::OpenMPOffloadMappingFlags &Flags,
12561 omp::OpenMPOffloadMappingFlags MemberOfFlag) {
12562 // If the entry is PTR_AND_OBJ but has not been marked with the special
12563 // placeholder value 0xFFFF in the MEMBER_OF field, then it should not be
12564 // marked as MEMBER_OF.
12565 if (static_cast<std::underlying_type_t<omp::OpenMPOffloadMappingFlags>>(
12566 Flags & omp::OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ) &&
12567 static_cast<std::underlying_type_t<omp::OpenMPOffloadMappingFlags>>(
12568 (Flags & omp::OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) !=
12569 omp::OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF))
12570 return;
12571
12572 // Entries with ATTACH are not members-of anything. They are handled
12573 // separately by the runtime after other maps have been handled.
12574 if (static_cast<std::underlying_type_t<omp::OpenMPOffloadMappingFlags>>(
12575 Flags & omp::OpenMPOffloadMappingFlags::OMP_MAP_ATTACH))
12576 return;
12577
12578 // Reset the placeholder value to prepare the flag for the assignment of the
12579 // proper MEMBER_OF value.
12580 Flags &= ~omp::OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF;
12581 Flags |= MemberOfFlag;
12582}
12583
12584Constant *OpenMPIRBuilder::getAddrOfDeclareTargetVar(
12585 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind CaptureClause,
12586 OffloadEntriesInfoManager::OMPTargetDeviceClauseKind DeviceClause,
12587 bool IsDeclaration, bool IsExternallyVisible,
12588 TargetRegionEntryInfo EntryInfo, StringRef MangledName,
12589 std::vector<GlobalVariable *> &GeneratedRefs, bool OpenMPSIMD,
12590 std::vector<Triple> TargetTriple, Type *LlvmPtrTy,
12591 std::function<Constant *()> GlobalInitializer,
12592 std::function<GlobalValue::LinkageTypes()> VariableLinkage) {
12593 // TODO: convert this to utilise the IRBuilder Config rather than
12594 // a passed down argument.
12595 if (OpenMPSIMD)
12596 return nullptr;
12597
12598 if (CaptureClause == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink ||
12599 ((CaptureClause == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo ||
12600 CaptureClause ==
12601 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter) &&
12602 Config.hasRequiresUnifiedSharedMemory())) {
12603 SmallString<64> PtrName;
12604 {
12605 raw_svector_ostream OS(PtrName);
12606 OS << MangledName;
12607 if (!IsExternallyVisible)
12608 OS << format(Fmt: "_%x", Vals: EntryInfo.FileID);
12609 OS << "_decl_tgt_ref_ptr";
12610 }
12611
12612 Value *Ptr = M.getNamedValue(Name: PtrName);
12613
12614 if (!Ptr) {
12615 GlobalValue *GlobalValue = M.getNamedValue(Name: MangledName);
12616 Ptr = getOrCreateInternalVariable(Ty: LlvmPtrTy, Name: PtrName);
12617
12618 auto *GV = cast<GlobalVariable>(Val: Ptr);
12619 GV->setLinkage(GlobalValue::WeakAnyLinkage);
12620
12621 if (!Config.isTargetDevice()) {
12622 if (GlobalInitializer)
12623 GV->setInitializer(GlobalInitializer());
12624 else
12625 GV->setInitializer(GlobalValue);
12626 }
12627
12628 registerTargetGlobalVariable(
12629 CaptureClause, DeviceClause, IsDeclaration, IsExternallyVisible,
12630 EntryInfo, MangledName, GeneratedRefs, OpenMPSIMD, TargetTriple,
12631 GlobalInitializer, VariableLinkage, LlvmPtrTy, Addr: cast<Constant>(Val: Ptr));
12632 }
12633
12634 return cast<Constant>(Val: Ptr);
12635 }
12636
12637 return nullptr;
12638}
12639
12640void OpenMPIRBuilder::registerTargetGlobalVariable(
12641 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind CaptureClause,
12642 OffloadEntriesInfoManager::OMPTargetDeviceClauseKind DeviceClause,
12643 bool IsDeclaration, bool IsExternallyVisible,
12644 TargetRegionEntryInfo EntryInfo, StringRef MangledName,
12645 std::vector<GlobalVariable *> &GeneratedRefs, bool OpenMPSIMD,
12646 std::vector<Triple> TargetTriple,
12647 std::function<Constant *()> GlobalInitializer,
12648 std::function<GlobalValue::LinkageTypes()> VariableLinkage, Type *LlvmPtrTy,
12649 Constant *Addr) {
12650 if (DeviceClause != OffloadEntriesInfoManager::OMPTargetDeviceClauseAny ||
12651 (TargetTriple.empty() && !Config.isTargetDevice()))
12652 return;
12653
12654 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind Flags;
12655 StringRef VarName;
12656 int64_t VarSize;
12657 GlobalValue::LinkageTypes Linkage;
12658
12659 if ((CaptureClause == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo ||
12660 CaptureClause ==
12661 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter) &&
12662 !Config.hasRequiresUnifiedSharedMemory()) {
12663 Flags = OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo;
12664 VarName = MangledName;
12665 GlobalValue *LlvmVal = M.getNamedValue(Name: VarName);
12666
12667 if (!IsDeclaration)
12668 VarSize = divideCeil(
12669 Numerator: M.getDataLayout().getTypeSizeInBits(Ty: LlvmVal->getValueType()), Denominator: 8);
12670 else
12671 VarSize = 0;
12672 Linkage = (VariableLinkage) ? VariableLinkage() : LlvmVal->getLinkage();
12673
12674 // This is a workaround carried over from Clang which prevents undesired
12675 // optimisation of internal variables.
12676 if (Config.isTargetDevice() &&
12677 (!IsExternallyVisible || Linkage == GlobalValue::LinkOnceODRLinkage)) {
12678 // Do not create a "ref-variable" if the original is not also available
12679 // on the host.
12680 if (!OffloadInfoManager.hasDeviceGlobalVarEntryInfo(VarName))
12681 return;
12682
12683 std::string RefName = createPlatformSpecificName(Parts: {VarName, "ref"});
12684
12685 if (!M.getNamedValue(Name: RefName)) {
12686 Constant *AddrRef =
12687 getOrCreateInternalVariable(Ty: Addr->getType(), Name: RefName);
12688 auto *GvAddrRef = cast<GlobalVariable>(Val: AddrRef);
12689 GvAddrRef->setConstant(true);
12690 GvAddrRef->setLinkage(GlobalValue::InternalLinkage);
12691 GvAddrRef->setInitializer(Addr);
12692 GeneratedRefs.push_back(x: GvAddrRef);
12693 }
12694 }
12695 } else {
12696 if (CaptureClause == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink)
12697 Flags = OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink;
12698 else
12699 Flags = OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo;
12700
12701 if (Config.isTargetDevice()) {
12702 VarName = (Addr) ? Addr->getName() : "";
12703 Addr = nullptr;
12704 } else {
12705 Addr = getAddrOfDeclareTargetVar(
12706 CaptureClause, DeviceClause, IsDeclaration, IsExternallyVisible,
12707 EntryInfo, MangledName, GeneratedRefs, OpenMPSIMD, TargetTriple,
12708 LlvmPtrTy, GlobalInitializer, VariableLinkage);
12709 VarName = (Addr) ? Addr->getName() : "";
12710 }
12711 VarSize = M.getDataLayout().getPointerSize();
12712 Linkage = GlobalValue::WeakAnyLinkage;
12713 }
12714
12715 OffloadInfoManager.registerDeviceGlobalVarEntryInfo(VarName, Addr, VarSize,
12716 Flags, Linkage);
12717}
12718
12719/// Loads all the offload entries information from the host IR
12720/// metadata.
12721void OpenMPIRBuilder::loadOffloadInfoMetadata(Module &M) {
12722 // If we are in target mode, load the metadata from the host IR. This code has
12723 // to match the metadata creation in createOffloadEntriesAndInfoMetadata().
12724
12725 NamedMDNode *MD = M.getNamedMetadata(Name: ompOffloadInfoName);
12726 if (!MD)
12727 return;
12728
12729 for (MDNode *MN : MD->operands()) {
12730 auto &&GetMDInt = [MN](unsigned Idx) {
12731 auto *V = cast<ConstantAsMetadata>(Val: MN->getOperand(I: Idx));
12732 return cast<ConstantInt>(Val: V->getValue())->getZExtValue();
12733 };
12734
12735 auto &&GetMDString = [MN](unsigned Idx) {
12736 auto *V = cast<MDString>(Val: MN->getOperand(I: Idx));
12737 return V->getString();
12738 };
12739
12740 switch (GetMDInt(0)) {
12741 default:
12742 llvm_unreachable("Unexpected metadata!");
12743 break;
12744 case OffloadEntriesInfoManager::OffloadEntryInfo::
12745 OffloadingEntryInfoTargetRegion: {
12746 TargetRegionEntryInfo EntryInfo(/*ParentName=*/GetMDString(3),
12747 /*DeviceID=*/GetMDInt(1),
12748 /*FileID=*/GetMDInt(2),
12749 /*Line=*/GetMDInt(4),
12750 /*Count=*/GetMDInt(5));
12751 OffloadInfoManager.initializeTargetRegionEntryInfo(EntryInfo,
12752 /*Order=*/GetMDInt(6));
12753 break;
12754 }
12755 case OffloadEntriesInfoManager::OffloadEntryInfo::
12756 OffloadingEntryInfoDeviceGlobalVar:
12757 OffloadInfoManager.initializeDeviceGlobalVarEntryInfo(
12758 /*MangledName=*/Name: GetMDString(1),
12759 Flags: static_cast<OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind>(
12760 /*Flags=*/GetMDInt(2)),
12761 /*Order=*/GetMDInt(3));
12762 break;
12763 }
12764 }
12765}
12766
12767void OpenMPIRBuilder::loadOffloadInfoMetadata(vfs::FileSystem &VFS,
12768 StringRef HostFilePath) {
12769 if (HostFilePath.empty())
12770 return;
12771
12772 auto Buf = VFS.getBufferForFile(Name: HostFilePath);
12773 if (std::error_code Err = Buf.getError()) {
12774 report_fatal_error(reason: ("error opening host file from host file path inside of "
12775 "OpenMPIRBuilder: " +
12776 Err.message())
12777 .c_str());
12778 }
12779
12780 LLVMContext Ctx;
12781 auto M = expectedToErrorOrAndEmitErrors(
12782 Ctx, Val: parseBitcodeFile(Buffer: Buf.get()->getMemBufferRef(), Context&: Ctx));
12783 if (std::error_code Err = M.getError()) {
12784 report_fatal_error(
12785 reason: ("error parsing host file inside of OpenMPIRBuilder: " + Err.message())
12786 .c_str());
12787 }
12788
12789 loadOffloadInfoMetadata(M&: *M.get());
12790}
12791
12792OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createIteratorLoop(
12793 LocationDescription Loc, llvm::Value *TripCount, IteratorBodyGenTy BodyGen,
12794 llvm::StringRef Name) {
12795 Builder.restoreIP(IP: Loc.IP);
12796
12797 BasicBlock *CurBB = Builder.GetInsertBlock();
12798 assert(CurBB &&
12799 "expected a valid insertion block for creating an iterator loop");
12800 Function *F = CurBB->getParent();
12801
12802 InsertPointTy SplitIP = Builder.saveIP();
12803 if (SplitIP.getPoint() == CurBB->end())
12804 if (Instruction *Terminator = CurBB->getTerminatorOrNull())
12805 SplitIP = InsertPointTy(CurBB, Terminator->getIterator());
12806
12807 BasicBlock *ContBB =
12808 splitBB(IP: SplitIP, /*CreateBranch=*/false,
12809 DL: Builder.getCurrentDebugLocation(), Name: "omp.it.cont");
12810
12811 CanonicalLoopInfo *CLI =
12812 createLoopSkeleton(DL: Builder.getCurrentDebugLocation(), TripCount, F,
12813 /*PreInsertBefore=*/ContBB,
12814 /*PostInsertBefore=*/ContBB, Name);
12815
12816 // Enter loop from original block.
12817 redirectTo(Source: CurBB, Target: CLI->getPreheader(), DL: Builder.getCurrentDebugLocation());
12818
12819 // Remove the unconditional branch inserted by createLoopSkeleton in the body
12820 if (Instruction *T = CLI->getBody()->getTerminatorOrNull())
12821 T->eraseFromParent();
12822
12823 InsertPointTy BodyIP = CLI->getBodyIP();
12824 if (llvm::Error Err = BodyGen(BodyIP, CLI->getIndVar()))
12825 return Err;
12826
12827 // Body must either fallthrough to the latch or branch directly to it.
12828 if (Instruction *BodyTerminator = CLI->getBody()->getTerminatorOrNull()) {
12829 auto *BodyBr = dyn_cast<UncondBrInst>(Val: BodyTerminator);
12830 if (!BodyBr || BodyBr->getSuccessor() != CLI->getLatch()) {
12831 return make_error<StringError>(
12832 Args: "iterator bodygen must terminate the canonical body with an "
12833 "unconditional branch to the loop latch",
12834 Args: inconvertibleErrorCode());
12835 }
12836 } else {
12837 // Ensure we end the loop body by jumping to the latch.
12838 Builder.SetInsertPoint(CLI->getBody());
12839 Builder.CreateBr(Dest: CLI->getLatch());
12840 }
12841
12842 // Link After -> ContBB
12843 Builder.SetInsertPoint(TheBB: CLI->getAfter(), IP: CLI->getAfter()->begin());
12844 if (!CLI->getAfter()->hasTerminator())
12845 Builder.CreateBr(Dest: ContBB);
12846
12847 return InsertPointTy{ContBB, ContBB->begin()};
12848}
12849
12850/// Mangle the parameter part of the vector function name according to
12851/// their OpenMP classification. The mangling function is defined in
12852/// section 4.5 of the AAVFABI(2021Q1).
12853static std::string mangleVectorParameters(
12854 ArrayRef<llvm::OpenMPIRBuilder::DeclareSimdAttrTy> ParamAttrs) {
12855 SmallString<256> Buffer;
12856 llvm::raw_svector_ostream Out(Buffer);
12857 for (const auto &ParamAttr : ParamAttrs) {
12858 switch (ParamAttr.Kind) {
12859 case llvm::OpenMPIRBuilder::DeclareSimdKindTy::Linear:
12860 Out << 'l';
12861 break;
12862 case llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearRef:
12863 Out << 'R';
12864 break;
12865 case llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearUVal:
12866 Out << 'U';
12867 break;
12868 case llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearVal:
12869 Out << 'L';
12870 break;
12871 case llvm::OpenMPIRBuilder::DeclareSimdKindTy::Uniform:
12872 Out << 'u';
12873 break;
12874 case llvm::OpenMPIRBuilder::DeclareSimdKindTy::Vector:
12875 Out << 'v';
12876 break;
12877 }
12878 if (ParamAttr.HasVarStride)
12879 Out << "s" << ParamAttr.StrideOrArg;
12880 else if (ParamAttr.Kind ==
12881 llvm::OpenMPIRBuilder::DeclareSimdKindTy::Linear ||
12882 ParamAttr.Kind ==
12883 llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearRef ||
12884 ParamAttr.Kind ==
12885 llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearUVal ||
12886 ParamAttr.Kind ==
12887 llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearVal) {
12888 // Don't print the step value if it is not present or if it is
12889 // equal to 1.
12890 if (ParamAttr.StrideOrArg < 0)
12891 Out << 'n' << -ParamAttr.StrideOrArg;
12892 else if (ParamAttr.StrideOrArg != 1)
12893 Out << ParamAttr.StrideOrArg;
12894 }
12895
12896 if (!!ParamAttr.Alignment)
12897 Out << 'a' << ParamAttr.Alignment;
12898 }
12899
12900 return std::string(Out.str());
12901}
12902
12903void OpenMPIRBuilder::emitX86DeclareSimdFunction(
12904 llvm::Function *Fn, unsigned NumElts, const llvm::APSInt &VLENVal,
12905 llvm::ArrayRef<DeclareSimdAttrTy> ParamAttrs, DeclareSimdBranch Branch) {
12906 struct ISADataTy {
12907 char ISA;
12908 unsigned VecRegSize;
12909 };
12910 ISADataTy ISAData[] = {
12911 {.ISA: 'b', .VecRegSize: 128}, // SSE
12912 {.ISA: 'c', .VecRegSize: 256}, // AVX
12913 {.ISA: 'd', .VecRegSize: 256}, // AVX2
12914 {.ISA: 'e', .VecRegSize: 512}, // AVX512
12915 };
12916 llvm::SmallVector<char, 2> Masked;
12917 switch (Branch) {
12918 case DeclareSimdBranch::Undefined:
12919 Masked.push_back(Elt: 'N');
12920 Masked.push_back(Elt: 'M');
12921 break;
12922 case DeclareSimdBranch::Notinbranch:
12923 Masked.push_back(Elt: 'N');
12924 break;
12925 case DeclareSimdBranch::Inbranch:
12926 Masked.push_back(Elt: 'M');
12927 break;
12928 }
12929 for (char Mask : Masked) {
12930 for (const ISADataTy &Data : ISAData) {
12931 llvm::SmallString<256> Buffer;
12932 llvm::raw_svector_ostream Out(Buffer);
12933 Out << "_ZGV" << Data.ISA << Mask;
12934 if (!VLENVal) {
12935 assert(NumElts && "Non-zero simdlen/cdtsize expected");
12936 Out << llvm::APSInt::getUnsigned(X: Data.VecRegSize / NumElts);
12937 } else {
12938 Out << VLENVal;
12939 }
12940 Out << mangleVectorParameters(ParamAttrs);
12941 Out << '_' << Fn->getName();
12942 Fn->addFnAttr(Kind: Out.str());
12943 }
12944 }
12945}
12946
12947// Function used to add the attribute. The parameter `VLEN` is templated to
12948// allow the use of `x` when targeting scalable functions for SVE.
12949template <typename T>
12950static void addAArch64VectorName(T VLEN, StringRef LMask, StringRef Prefix,
12951 char ISA, StringRef ParSeq,
12952 StringRef MangledName, bool OutputBecomesInput,
12953 llvm::Function *Fn) {
12954 SmallString<256> Buffer;
12955 llvm::raw_svector_ostream Out(Buffer);
12956 Out << Prefix << ISA << LMask << VLEN;
12957 if (OutputBecomesInput)
12958 Out << 'v';
12959 Out << ParSeq << '_' << MangledName;
12960 Fn->addFnAttr(Kind: Out.str());
12961}
12962
12963// Helper function to generate the Advanced SIMD names depending on the value
12964// of the NDS when simdlen is not present.
12965static void addAArch64AdvSIMDNDSNames(unsigned NDS, StringRef Mask,
12966 StringRef Prefix, char ISA,
12967 StringRef ParSeq, StringRef MangledName,
12968 bool OutputBecomesInput,
12969 llvm::Function *Fn) {
12970 switch (NDS) {
12971 case 8:
12972 addAArch64VectorName(VLEN: 8, LMask: Mask, Prefix, ISA, ParSeq, MangledName,
12973 OutputBecomesInput, Fn);
12974 addAArch64VectorName(VLEN: 16, LMask: Mask, Prefix, ISA, ParSeq, MangledName,
12975 OutputBecomesInput, Fn);
12976 break;
12977 case 16:
12978 addAArch64VectorName(VLEN: 4, LMask: Mask, Prefix, ISA, ParSeq, MangledName,
12979 OutputBecomesInput, Fn);
12980 addAArch64VectorName(VLEN: 8, LMask: Mask, Prefix, ISA, ParSeq, MangledName,
12981 OutputBecomesInput, Fn);
12982 break;
12983 case 32:
12984 addAArch64VectorName(VLEN: 2, LMask: Mask, Prefix, ISA, ParSeq, MangledName,
12985 OutputBecomesInput, Fn);
12986 addAArch64VectorName(VLEN: 4, LMask: Mask, Prefix, ISA, ParSeq, MangledName,
12987 OutputBecomesInput, Fn);
12988 break;
12989 case 64:
12990 case 128:
12991 addAArch64VectorName(VLEN: 2, LMask: Mask, Prefix, ISA, ParSeq, MangledName,
12992 OutputBecomesInput, Fn);
12993 break;
12994 default:
12995 llvm_unreachable("Scalar type is too wide.");
12996 }
12997}
12998
12999/// Emit vector function attributes for AArch64, as defined in the AAVFABI.
13000void OpenMPIRBuilder::emitAArch64DeclareSimdFunction(
13001 llvm::Function *Fn, unsigned UserVLEN,
13002 llvm::ArrayRef<DeclareSimdAttrTy> ParamAttrs, DeclareSimdBranch Branch,
13003 char ISA, unsigned NarrowestDataSize, bool OutputBecomesInput) {
13004 assert((ISA == 'n' || ISA == 's') && "Expected ISA either 's' or 'n'.");
13005
13006 // Sort out parameter sequence.
13007 const std::string ParSeq = mangleVectorParameters(ParamAttrs);
13008 StringRef Prefix = "_ZGV";
13009 StringRef MangledName = Fn->getName();
13010
13011 // Generate simdlen from user input (if any).
13012 if (UserVLEN) {
13013 if (ISA == 's') {
13014 // SVE generates only a masked function.
13015 addAArch64VectorName(VLEN: UserVLEN, LMask: "M", Prefix, ISA, ParSeq, MangledName,
13016 OutputBecomesInput, Fn);
13017 return;
13018 }
13019
13020 switch (Branch) {
13021 case DeclareSimdBranch::Undefined:
13022 addAArch64VectorName(VLEN: UserVLEN, LMask: "N", Prefix, ISA, ParSeq, MangledName,
13023 OutputBecomesInput, Fn);
13024 addAArch64VectorName(VLEN: UserVLEN, LMask: "M", Prefix, ISA, ParSeq, MangledName,
13025 OutputBecomesInput, Fn);
13026 break;
13027 case DeclareSimdBranch::Inbranch:
13028 addAArch64VectorName(VLEN: UserVLEN, LMask: "M", Prefix, ISA, ParSeq, MangledName,
13029 OutputBecomesInput, Fn);
13030 break;
13031 case DeclareSimdBranch::Notinbranch:
13032 addAArch64VectorName(VLEN: UserVLEN, LMask: "N", Prefix, ISA, ParSeq, MangledName,
13033 OutputBecomesInput, Fn);
13034 break;
13035 }
13036 return;
13037 }
13038
13039 if (ISA == 's') {
13040 // SVE, section 3.4.1, item 1.
13041 addAArch64VectorName(VLEN: "x", LMask: "M", Prefix, ISA, ParSeq, MangledName,
13042 OutputBecomesInput, Fn);
13043 return;
13044 }
13045
13046 switch (Branch) {
13047 case DeclareSimdBranch::Undefined:
13048 addAArch64AdvSIMDNDSNames(NDS: NarrowestDataSize, Mask: "N", Prefix, ISA, ParSeq,
13049 MangledName, OutputBecomesInput, Fn);
13050 addAArch64AdvSIMDNDSNames(NDS: NarrowestDataSize, Mask: "M", Prefix, ISA, ParSeq,
13051 MangledName, OutputBecomesInput, Fn);
13052 break;
13053 case DeclareSimdBranch::Inbranch:
13054 addAArch64AdvSIMDNDSNames(NDS: NarrowestDataSize, Mask: "M", Prefix, ISA, ParSeq,
13055 MangledName, OutputBecomesInput, Fn);
13056 break;
13057 case DeclareSimdBranch::Notinbranch:
13058 addAArch64AdvSIMDNDSNames(NDS: NarrowestDataSize, Mask: "N", Prefix, ISA, ParSeq,
13059 MangledName, OutputBecomesInput, Fn);
13060 break;
13061 }
13062}
13063
13064//===----------------------------------------------------------------------===//
13065// OffloadEntriesInfoManager
13066//===----------------------------------------------------------------------===//
13067
13068bool OffloadEntriesInfoManager::empty() const {
13069 return OffloadEntriesTargetRegion.empty() &&
13070 OffloadEntriesDeviceGlobalVar.empty();
13071}
13072
13073unsigned OffloadEntriesInfoManager::getTargetRegionEntryInfoCount(
13074 const TargetRegionEntryInfo &EntryInfo) const {
13075 auto It = OffloadEntriesTargetRegionCount.find(
13076 x: getTargetRegionEntryCountKey(EntryInfo));
13077 if (It == OffloadEntriesTargetRegionCount.end())
13078 return 0;
13079 return It->second;
13080}
13081
13082void OffloadEntriesInfoManager::incrementTargetRegionEntryInfoCount(
13083 const TargetRegionEntryInfo &EntryInfo) {
13084 OffloadEntriesTargetRegionCount[getTargetRegionEntryCountKey(EntryInfo)] =
13085 EntryInfo.Count + 1;
13086}
13087
13088/// Initialize target region entry.
13089void OffloadEntriesInfoManager::initializeTargetRegionEntryInfo(
13090 const TargetRegionEntryInfo &EntryInfo, unsigned Order) {
13091 OffloadEntriesTargetRegion[EntryInfo] =
13092 OffloadEntryInfoTargetRegion(Order, /*Addr=*/nullptr, /*ID=*/nullptr,
13093 OMPTargetRegionEntryTargetRegion);
13094 ++OffloadingEntriesNum;
13095}
13096
13097void OffloadEntriesInfoManager::registerTargetRegionEntryInfo(
13098 TargetRegionEntryInfo EntryInfo, Constant *Addr, Constant *ID,
13099 OMPTargetRegionEntryKind Flags) {
13100 assert(EntryInfo.Count == 0 && "expected default EntryInfo");
13101
13102 // Update the EntryInfo with the next available count for this location.
13103 EntryInfo.Count = getTargetRegionEntryInfoCount(EntryInfo);
13104
13105 // If we are emitting code for a target, the entry is already initialized,
13106 // only has to be registered.
13107 if (OMPBuilder->Config.isTargetDevice()) {
13108 // This could happen if the device compilation is invoked standalone.
13109 if (!hasTargetRegionEntryInfo(EntryInfo)) {
13110 return;
13111 }
13112 auto &Entry = OffloadEntriesTargetRegion[EntryInfo];
13113 Entry.setAddress(Addr);
13114 Entry.setID(ID);
13115 Entry.setFlags(Flags);
13116 } else {
13117 if (Flags == OffloadEntriesInfoManager::OMPTargetRegionEntryTargetRegion &&
13118 hasTargetRegionEntryInfo(EntryInfo, /*IgnoreAddressId*/ true))
13119 return;
13120 assert(!hasTargetRegionEntryInfo(EntryInfo) &&
13121 "Target region entry already registered!");
13122 OffloadEntryInfoTargetRegion Entry(OffloadingEntriesNum, Addr, ID, Flags);
13123 OffloadEntriesTargetRegion[EntryInfo] = Entry;
13124 ++OffloadingEntriesNum;
13125 }
13126 incrementTargetRegionEntryInfoCount(EntryInfo);
13127}
13128
13129bool OffloadEntriesInfoManager::hasTargetRegionEntryInfo(
13130 TargetRegionEntryInfo EntryInfo, bool IgnoreAddressId) const {
13131
13132 // Update the EntryInfo with the next available count for this location.
13133 EntryInfo.Count = getTargetRegionEntryInfoCount(EntryInfo);
13134
13135 auto It = OffloadEntriesTargetRegion.find(x: EntryInfo);
13136 if (It == OffloadEntriesTargetRegion.end()) {
13137 return false;
13138 }
13139 // Fail if this entry is already registered.
13140 if (!IgnoreAddressId && (It->second.getAddress() || It->second.getID()))
13141 return false;
13142 return true;
13143}
13144
13145void OffloadEntriesInfoManager::actOnTargetRegionEntriesInfo(
13146 const OffloadTargetRegionEntryInfoActTy &Action) {
13147 // Scan all target region entries and perform the provided action.
13148 for (const auto &It : OffloadEntriesTargetRegion) {
13149 Action(It.first, It.second);
13150 }
13151}
13152
13153void OffloadEntriesInfoManager::initializeDeviceGlobalVarEntryInfo(
13154 StringRef Name, OMPTargetGlobalVarEntryKind Flags, unsigned Order) {
13155 OffloadEntriesDeviceGlobalVar.try_emplace(Key: Name, Args&: Order, Args&: Flags);
13156 ++OffloadingEntriesNum;
13157}
13158
13159void OffloadEntriesInfoManager::registerDeviceGlobalVarEntryInfo(
13160 StringRef VarName, Constant *Addr, int64_t VarSize,
13161 OMPTargetGlobalVarEntryKind Flags, GlobalValue::LinkageTypes Linkage) {
13162 if (OMPBuilder->Config.isTargetDevice()) {
13163 // This could happen if the device compilation is invoked standalone.
13164 if (!hasDeviceGlobalVarEntryInfo(VarName))
13165 return;
13166 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
13167 if (Entry.getAddress() && hasDeviceGlobalVarEntryInfo(VarName)) {
13168 if (Entry.getVarSize() == 0) {
13169 Entry.setVarSize(VarSize);
13170 Entry.setLinkage(Linkage);
13171 }
13172 return;
13173 }
13174 Entry.setVarSize(VarSize);
13175 Entry.setLinkage(Linkage);
13176 Entry.setAddress(Addr);
13177 } else {
13178 if (hasDeviceGlobalVarEntryInfo(VarName)) {
13179 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
13180 assert(Entry.isValid() && Entry.getFlags() == Flags &&
13181 "Entry not initialized!");
13182 if (Entry.getVarSize() == 0) {
13183 Entry.setVarSize(VarSize);
13184 Entry.setLinkage(Linkage);
13185 }
13186 return;
13187 }
13188 if (Flags == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect ||
13189 Flags ==
13190 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirectVTable)
13191 OffloadEntriesDeviceGlobalVar.try_emplace(Key: VarName, Args&: OffloadingEntriesNum,
13192 Args&: Addr, Args&: VarSize, Args&: Flags, Args&: Linkage,
13193 Args: VarName.str());
13194 else
13195 OffloadEntriesDeviceGlobalVar.try_emplace(
13196 Key: VarName, Args&: OffloadingEntriesNum, Args&: Addr, Args&: VarSize, Args&: Flags, Args&: Linkage, Args: "");
13197 ++OffloadingEntriesNum;
13198 }
13199}
13200
13201void OffloadEntriesInfoManager::actOnDeviceGlobalVarEntriesInfo(
13202 const OffloadDeviceGlobalVarEntryInfoActTy &Action) {
13203 // Scan all target region entries and perform the provided action.
13204 for (const auto &E : OffloadEntriesDeviceGlobalVar)
13205 Action(E.getKey(), E.getValue());
13206}
13207
13208//===----------------------------------------------------------------------===//
13209// CanonicalLoopInfo
13210//===----------------------------------------------------------------------===//
13211
13212void CanonicalLoopInfo::collectControlBlocks(
13213 SmallVectorImpl<BasicBlock *> &BBs) {
13214 // We only count those BBs as control block for which we do not need to
13215 // reverse the CFG, i.e. not the loop body which can contain arbitrary control
13216 // flow. For consistency, this also means we do not add the Body block, which
13217 // is just the entry to the body code.
13218 BBs.reserve(N: BBs.size() + 6);
13219 BBs.append(IL: {getPreheader(), Header, Cond, Latch, Exit, getAfter()});
13220}
13221
13222BasicBlock *CanonicalLoopInfo::getPreheader() const {
13223 assert(isValid() && "Requires a valid canonical loop");
13224 for (BasicBlock *Pred : predecessors(BB: Header)) {
13225 if (Pred != Latch)
13226 return Pred;
13227 }
13228 llvm_unreachable("Missing preheader");
13229}
13230
13231void CanonicalLoopInfo::setTripCount(Value *TripCount) {
13232 assert(isValid() && "Requires a valid canonical loop");
13233
13234 Instruction *CmpI = &getCond()->front();
13235 assert(isa<CmpInst>(CmpI) && "First inst must compare IV with TripCount");
13236 CmpI->setOperand(i: 1, Val: TripCount);
13237
13238#ifndef NDEBUG
13239 assertOK();
13240#endif
13241}
13242
13243void CanonicalLoopInfo::mapIndVar(
13244 llvm::function_ref<Value *(Instruction *)> Updater) {
13245 assert(isValid() && "Requires a valid canonical loop");
13246
13247 Instruction *OldIV = getIndVar();
13248
13249 // Record all uses excluding those introduced by the updater. Uses by the
13250 // CanonicalLoopInfo itself to keep track of the number of iterations are
13251 // excluded.
13252 SmallVector<Use *> ReplacableUses;
13253 for (Use &U : OldIV->uses()) {
13254 auto *User = dyn_cast<Instruction>(Val: U.getUser());
13255 if (!User)
13256 continue;
13257 if (User->getParent() == getCond())
13258 continue;
13259 if (User->getParent() == getLatch())
13260 continue;
13261 ReplacableUses.push_back(Elt: &U);
13262 }
13263
13264 // Run the updater that may introduce new uses
13265 Value *NewIV = Updater(OldIV);
13266
13267 // Replace the old uses with the value returned by the updater.
13268 for (Use *U : ReplacableUses)
13269 U->set(NewIV);
13270
13271#ifndef NDEBUG
13272 assertOK();
13273#endif
13274}
13275
13276void CanonicalLoopInfo::assertOK() const {
13277#ifndef NDEBUG
13278 // No constraints if this object currently does not describe a loop.
13279 if (!isValid())
13280 return;
13281
13282 BasicBlock *Preheader = getPreheader();
13283 BasicBlock *Body = getBody();
13284 BasicBlock *After = getAfter();
13285
13286 // Verify standard control-flow we use for OpenMP loops.
13287 assert(Preheader);
13288 assert(isa<UncondBrInst>(Preheader->getTerminator()) &&
13289 "Preheader must terminate with unconditional branch");
13290 assert(Preheader->getSingleSuccessor() == Header &&
13291 "Preheader must jump to header");
13292
13293 assert(Header);
13294 assert(isa<UncondBrInst>(Header->getTerminator()) &&
13295 "Header must terminate with unconditional branch");
13296 assert(Header->getSingleSuccessor() == Cond &&
13297 "Header must jump to exiting block");
13298
13299 assert(Cond);
13300 assert(Cond->getSinglePredecessor() == Header &&
13301 "Exiting block only reachable from header");
13302
13303 assert(isa<CondBrInst>(Cond->getTerminator()) &&
13304 "Exiting block must terminate with conditional branch");
13305 assert(cast<CondBrInst>(Cond->getTerminator())->getSuccessor(0) == Body &&
13306 "Exiting block's first successor jump to the body");
13307 assert(cast<CondBrInst>(Cond->getTerminator())->getSuccessor(1) == Exit &&
13308 "Exiting block's second successor must exit the loop");
13309
13310 assert(Body);
13311 assert(Body->getSinglePredecessor() == Cond &&
13312 "Body only reachable from exiting block");
13313 assert(!isa<PHINode>(Body->front()));
13314
13315 assert(Latch);
13316 assert(isa<UncondBrInst>(Latch->getTerminator()) &&
13317 "Latch must terminate with unconditional branch");
13318 assert(Latch->getSingleSuccessor() == Header && "Latch must jump to header");
13319 // TODO: To support simple redirecting of the end of the body code that has
13320 // multiple; introduce another auxiliary basic block like preheader and after.
13321 assert(Latch->getSinglePredecessor() != nullptr);
13322 assert(!isa<PHINode>(Latch->front()));
13323
13324 assert(Exit);
13325 assert(isa<UncondBrInst>(Exit->getTerminator()) &&
13326 "Exit block must terminate with unconditional branch");
13327 assert(Exit->getSingleSuccessor() == After &&
13328 "Exit block must jump to after block");
13329
13330 assert(After);
13331 assert(After->getSinglePredecessor() == Exit &&
13332 "After block only reachable from exit block");
13333 assert(After->empty() || !isa<PHINode>(After->front()));
13334
13335 Instruction *IndVar = getIndVar();
13336 assert(IndVar && "Canonical induction variable not found?");
13337 assert(isa<IntegerType>(IndVar->getType()) &&
13338 "Induction variable must be an integer");
13339 assert(cast<PHINode>(IndVar)->getParent() == Header &&
13340 "Induction variable must be a PHI in the loop header");
13341 assert(cast<PHINode>(IndVar)->getIncomingBlock(0) == Preheader);
13342 assert(
13343 cast<ConstantInt>(cast<PHINode>(IndVar)->getIncomingValue(0))->isZero());
13344 assert(cast<PHINode>(IndVar)->getIncomingBlock(1) == Latch);
13345
13346 auto *NextIndVar = cast<PHINode>(IndVar)->getIncomingValue(1);
13347 assert(cast<Instruction>(NextIndVar)->getParent() == Latch);
13348 assert(cast<BinaryOperator>(NextIndVar)->getOpcode() == BinaryOperator::Add);
13349 assert(cast<BinaryOperator>(NextIndVar)->getOperand(0) == IndVar);
13350 assert(cast<ConstantInt>(cast<BinaryOperator>(NextIndVar)->getOperand(1))
13351 ->isOne());
13352
13353 Value *TripCount = getTripCount();
13354 assert(TripCount && "Loop trip count not found?");
13355 assert(IndVar->getType() == TripCount->getType() &&
13356 "Trip count and induction variable must have the same type");
13357
13358 auto *CmpI = cast<CmpInst>(&Cond->front());
13359 assert(CmpI->getPredicate() == CmpInst::ICMP_ULT &&
13360 "Exit condition must be a signed less-than comparison");
13361 assert(CmpI->getOperand(0) == IndVar &&
13362 "Exit condition must compare the induction variable");
13363 assert(CmpI->getOperand(1) == TripCount &&
13364 "Exit condition must compare with the trip count");
13365#endif
13366}
13367
13368void CanonicalLoopInfo::invalidate() {
13369 Header = nullptr;
13370 Cond = nullptr;
13371 Latch = nullptr;
13372 Exit = nullptr;
13373}
13374