1//===- OpenMPIRBuilder.cpp - Builder for LLVM-IR for OpenMP directives ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9///
10/// This file implements the OpenMPIRBuilder class, which is used as a
11/// convenient way to create LLVM instructions for OpenMP directives.
12///
13//===----------------------------------------------------------------------===//
14
15#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
16#include "llvm/ADT/SmallBitVector.h"
17#include "llvm/ADT/SmallSet.h"
18#include "llvm/ADT/SmallVectorExtras.h"
19#include "llvm/ADT/StringExtras.h"
20#include "llvm/ADT/StringRef.h"
21#include "llvm/Analysis/AssumptionCache.h"
22#include "llvm/Analysis/CodeMetrics.h"
23#include "llvm/Analysis/LoopInfo.h"
24#include "llvm/Analysis/OptimizationRemarkEmitter.h"
25#include "llvm/Analysis/PostDominators.h"
26#include "llvm/Analysis/ScalarEvolution.h"
27#include "llvm/Analysis/TargetLibraryInfo.h"
28#include "llvm/Bitcode/BitcodeReader.h"
29#include "llvm/Frontend/Offloading/Utility.h"
30#include "llvm/Frontend/OpenMP/OMPGridValues.h"
31#include "llvm/IR/Attributes.h"
32#include "llvm/IR/BasicBlock.h"
33#include "llvm/IR/CFG.h"
34#include "llvm/IR/CallingConv.h"
35#include "llvm/IR/Constant.h"
36#include "llvm/IR/Constants.h"
37#include "llvm/IR/DIBuilder.h"
38#include "llvm/IR/DebugInfoMetadata.h"
39#include "llvm/IR/DerivedTypes.h"
40#include "llvm/IR/Function.h"
41#include "llvm/IR/GlobalVariable.h"
42#include "llvm/IR/IRBuilder.h"
43#include "llvm/IR/InstIterator.h"
44#include "llvm/IR/IntrinsicInst.h"
45#include "llvm/IR/LLVMContext.h"
46#include "llvm/IR/MDBuilder.h"
47#include "llvm/IR/Metadata.h"
48#include "llvm/IR/PassInstrumentation.h"
49#include "llvm/IR/PassManager.h"
50#include "llvm/IR/ReplaceConstant.h"
51#include "llvm/IR/Value.h"
52#include "llvm/MC/TargetRegistry.h"
53#include "llvm/Support/CommandLine.h"
54#include "llvm/Support/Error.h"
55#include "llvm/Support/ErrorHandling.h"
56#include "llvm/Support/FileSystem.h"
57#include "llvm/Support/NVVMAttributes.h"
58#include "llvm/Support/VirtualFileSystem.h"
59#include "llvm/Target/TargetMachine.h"
60#include "llvm/Target/TargetOptions.h"
61#include "llvm/Transforms/Utils/BasicBlockUtils.h"
62#include "llvm/Transforms/Utils/Cloning.h"
63#include "llvm/Transforms/Utils/CodeExtractor.h"
64#include "llvm/Transforms/Utils/LoopPeel.h"
65#include "llvm/Transforms/Utils/UnrollLoop.h"
66
67#include <cstdint>
68#include <optional>
69
70#define DEBUG_TYPE "openmp-ir-builder"
71
72using namespace llvm;
73using namespace omp;
74
75static cl::opt<bool>
76 OptimisticAttributes("openmp-ir-builder-optimistic-attributes", cl::Hidden,
77 cl::desc("Use optimistic attributes describing "
78 "'as-if' properties of runtime calls."),
79 cl::init(Val: false));
80
81static cl::opt<double> UnrollThresholdFactor(
82 "openmp-ir-builder-unroll-threshold-factor", cl::Hidden,
83 cl::desc("Factor for the unroll threshold to account for code "
84 "simplifications still taking place"),
85 cl::init(Val: 1.5));
86
87static cl::opt<bool> UseDefaultMaxThreads(
88 "openmp-ir-builder-use-default-max-threads", cl::Hidden,
89 cl::desc("Use a default max threads if none is provided."), cl::init(Val: true));
90
91#ifndef NDEBUG
92/// Return whether IP1 and IP2 are ambiguous, i.e. that inserting instructions
93/// at position IP1 may change the meaning of IP2 or vice-versa. This is because
94/// an InsertPoint stores the instruction before something is inserted. For
95/// instance, if both point to the same instruction, two IRBuilders alternating
96/// creating instruction will cause the instructions to be interleaved.
97static bool isConflictIP(IRBuilder<>::InsertPoint IP1,
98 IRBuilder<>::InsertPoint IP2) {
99 if (!IP1.isSet() || !IP2.isSet())
100 return false;
101 return IP1.getBlock() == IP2.getBlock() && IP1.getPoint() == IP2.getPoint();
102}
103
104static bool isValidWorkshareLoopScheduleType(OMPScheduleType SchedType) {
105 // Valid ordered/unordered and base algorithm combinations.
106 switch (SchedType & ~OMPScheduleType::MonotonicityMask) {
107 case OMPScheduleType::UnorderedStaticChunked:
108 case OMPScheduleType::UnorderedStatic:
109 case OMPScheduleType::UnorderedDynamicChunked:
110 case OMPScheduleType::UnorderedGuidedChunked:
111 case OMPScheduleType::UnorderedRuntime:
112 case OMPScheduleType::UnorderedAuto:
113 case OMPScheduleType::UnorderedTrapezoidal:
114 case OMPScheduleType::UnorderedGreedy:
115 case OMPScheduleType::UnorderedBalanced:
116 case OMPScheduleType::UnorderedGuidedIterativeChunked:
117 case OMPScheduleType::UnorderedGuidedAnalyticalChunked:
118 case OMPScheduleType::UnorderedSteal:
119 case OMPScheduleType::UnorderedStaticBalancedChunked:
120 case OMPScheduleType::UnorderedGuidedSimd:
121 case OMPScheduleType::UnorderedRuntimeSimd:
122 case OMPScheduleType::OrderedStaticChunked:
123 case OMPScheduleType::OrderedStatic:
124 case OMPScheduleType::OrderedDynamicChunked:
125 case OMPScheduleType::OrderedGuidedChunked:
126 case OMPScheduleType::OrderedRuntime:
127 case OMPScheduleType::OrderedAuto:
128 case OMPScheduleType::OrderdTrapezoidal:
129 case OMPScheduleType::NomergeUnorderedStaticChunked:
130 case OMPScheduleType::NomergeUnorderedStatic:
131 case OMPScheduleType::NomergeUnorderedDynamicChunked:
132 case OMPScheduleType::NomergeUnorderedGuidedChunked:
133 case OMPScheduleType::NomergeUnorderedRuntime:
134 case OMPScheduleType::NomergeUnorderedAuto:
135 case OMPScheduleType::NomergeUnorderedTrapezoidal:
136 case OMPScheduleType::NomergeUnorderedGreedy:
137 case OMPScheduleType::NomergeUnorderedBalanced:
138 case OMPScheduleType::NomergeUnorderedGuidedIterativeChunked:
139 case OMPScheduleType::NomergeUnorderedGuidedAnalyticalChunked:
140 case OMPScheduleType::NomergeUnorderedSteal:
141 case OMPScheduleType::NomergeOrderedStaticChunked:
142 case OMPScheduleType::NomergeOrderedStatic:
143 case OMPScheduleType::NomergeOrderedDynamicChunked:
144 case OMPScheduleType::NomergeOrderedGuidedChunked:
145 case OMPScheduleType::NomergeOrderedRuntime:
146 case OMPScheduleType::NomergeOrderedAuto:
147 case OMPScheduleType::NomergeOrderedTrapezoidal:
148 case OMPScheduleType::OrderedDistributeChunked:
149 case OMPScheduleType::OrderedDistribute:
150 break;
151 default:
152 return false;
153 }
154
155 // Must not set both monotonicity modifiers at the same time.
156 OMPScheduleType MonotonicityFlags =
157 SchedType & OMPScheduleType::MonotonicityMask;
158 if (MonotonicityFlags == OMPScheduleType::MonotonicityMask)
159 return false;
160
161 return true;
162}
163#endif
164
165/// This is wrapper over IRBuilderBase::restoreIP that also restores the current
166/// debug location to the last instruction in the specified basic block if the
167/// insert point points to the end of the block.
168static void restoreIPandDebugLoc(llvm::IRBuilderBase &Builder,
169 llvm::IRBuilderBase::InsertPoint IP) {
170 Builder.restoreIP(IP);
171 llvm::BasicBlock *BB = Builder.GetInsertBlock();
172 llvm::BasicBlock::iterator I = Builder.GetInsertPoint();
173 if (!BB->empty() && I == BB->end())
174 Builder.SetCurrentDebugLocation(BB->back().getStableDebugLoc());
175}
176
177static bool hasGridValue(const Triple &T) {
178 return T.isAMDGPU() || T.isNVPTX() || T.isSPIRV();
179}
180
181static const omp::GV &getGridValue(const Triple &T, Function *Kernel) {
182 if (T.isAMDGPU()) {
183 StringRef Features =
184 Kernel->getFnAttribute(Kind: "target-features").getValueAsString();
185 if (Features.count(Str: "+wavefrontsize64"))
186 return omp::getAMDGPUGridValues<64>();
187 return omp::getAMDGPUGridValues<32>();
188 }
189 if (T.isNVPTX())
190 return omp::NVPTXGridValues;
191 if (T.isSPIRV())
192 return omp::SPIRVGridValues;
193 llvm_unreachable("No grid value available for this architecture!");
194}
195
196/// Determine which scheduling algorithm to use, determined from schedule clause
197/// arguments.
198static OMPScheduleType
199getOpenMPBaseScheduleType(llvm::omp::ScheduleKind ClauseKind, bool HasChunks,
200 bool HasSimdModifier, bool HasDistScheduleChunks) {
201 // Currently, the default schedule it static.
202 switch (ClauseKind) {
203 case OMP_SCHEDULE_Default:
204 case OMP_SCHEDULE_Static:
205 return HasChunks ? OMPScheduleType::BaseStaticChunked
206 : OMPScheduleType::BaseStatic;
207 case OMP_SCHEDULE_Dynamic:
208 return OMPScheduleType::BaseDynamicChunked;
209 case OMP_SCHEDULE_Guided:
210 return HasSimdModifier ? OMPScheduleType::BaseGuidedSimd
211 : OMPScheduleType::BaseGuidedChunked;
212 case OMP_SCHEDULE_Auto:
213 return llvm::omp::OMPScheduleType::BaseAuto;
214 case OMP_SCHEDULE_Runtime:
215 return HasSimdModifier ? OMPScheduleType::BaseRuntimeSimd
216 : OMPScheduleType::BaseRuntime;
217 case OMP_SCHEDULE_Distribute:
218 return HasDistScheduleChunks ? OMPScheduleType::BaseDistributeChunked
219 : OMPScheduleType::BaseDistribute;
220 }
221 llvm_unreachable("unhandled schedule clause argument");
222}
223
224/// Adds ordering modifier flags to schedule type.
225static OMPScheduleType
226getOpenMPOrderingScheduleType(OMPScheduleType BaseScheduleType,
227 bool HasOrderedClause) {
228 assert((BaseScheduleType & OMPScheduleType::ModifierMask) ==
229 OMPScheduleType::None &&
230 "Must not have ordering nor monotonicity flags already set");
231
232 OMPScheduleType OrderingModifier = HasOrderedClause
233 ? OMPScheduleType::ModifierOrdered
234 : OMPScheduleType::ModifierUnordered;
235 OMPScheduleType OrderingScheduleType = BaseScheduleType | OrderingModifier;
236
237 // Unsupported combinations
238 if (OrderingScheduleType ==
239 (OMPScheduleType::BaseGuidedSimd | OMPScheduleType::ModifierOrdered))
240 return OMPScheduleType::OrderedGuidedChunked;
241 else if (OrderingScheduleType == (OMPScheduleType::BaseRuntimeSimd |
242 OMPScheduleType::ModifierOrdered))
243 return OMPScheduleType::OrderedRuntime;
244
245 return OrderingScheduleType;
246}
247
248/// Adds monotonicity modifier flags to schedule type.
249static OMPScheduleType
250getOpenMPMonotonicityScheduleType(OMPScheduleType ScheduleType,
251 bool HasSimdModifier, bool HasMonotonic,
252 bool HasNonmonotonic, bool HasOrderedClause) {
253 assert((ScheduleType & OMPScheduleType::MonotonicityMask) ==
254 OMPScheduleType::None &&
255 "Must not have monotonicity flags already set");
256 assert((!HasMonotonic || !HasNonmonotonic) &&
257 "Monotonic and Nonmonotonic are contradicting each other");
258
259 if (HasMonotonic) {
260 return ScheduleType | OMPScheduleType::ModifierMonotonic;
261 } else if (HasNonmonotonic) {
262 return ScheduleType | OMPScheduleType::ModifierNonmonotonic;
263 } else {
264 // OpenMP 5.1, 2.11.4 Worksharing-Loop Construct, Description.
265 // If the static schedule kind is specified or if the ordered clause is
266 // specified, and if the nonmonotonic modifier is not specified, the
267 // effect is as if the monotonic modifier is specified. Otherwise, unless
268 // the monotonic modifier is specified, the effect is as if the
269 // nonmonotonic modifier is specified.
270 OMPScheduleType BaseScheduleType =
271 ScheduleType & ~OMPScheduleType::ModifierMask;
272 if ((BaseScheduleType == OMPScheduleType::BaseStatic) ||
273 (BaseScheduleType == OMPScheduleType::BaseStaticChunked) ||
274 HasOrderedClause) {
275 // The monotonic is used by default in openmp runtime library, so no need
276 // to set it.
277 return ScheduleType;
278 } else {
279 return ScheduleType | OMPScheduleType::ModifierNonmonotonic;
280 }
281 }
282}
283
284/// Determine the schedule type using schedule and ordering clause arguments.
285static OMPScheduleType
286computeOpenMPScheduleType(ScheduleKind ClauseKind, bool HasChunks,
287 bool HasSimdModifier, bool HasMonotonicModifier,
288 bool HasNonmonotonicModifier, bool HasOrderedClause,
289 bool HasDistScheduleChunks) {
290 OMPScheduleType BaseSchedule = getOpenMPBaseScheduleType(
291 ClauseKind, HasChunks, HasSimdModifier, HasDistScheduleChunks);
292 OMPScheduleType OrderedSchedule =
293 getOpenMPOrderingScheduleType(BaseScheduleType: BaseSchedule, HasOrderedClause);
294 OMPScheduleType Result = getOpenMPMonotonicityScheduleType(
295 ScheduleType: OrderedSchedule, HasSimdModifier, HasMonotonic: HasMonotonicModifier,
296 HasNonmonotonic: HasNonmonotonicModifier, HasOrderedClause);
297
298 assert(isValidWorkshareLoopScheduleType(Result));
299 return Result;
300}
301
302/// Given a function, if it represents the entry point of a target kernel, this
303/// returns the execution mode flags associated with that kernel.
304static std::optional<omp::OMPTgtExecModeFlags>
305getTargetKernelExecMode(Function &Kernel) {
306 CallInst *TargetInitCall = nullptr;
307 for (Instruction &Inst : Kernel.getEntryBlock()) {
308 if (auto *Call = dyn_cast<CallInst>(Val: &Inst)) {
309 if (Call->getCalledFunction()->getName() == "__kmpc_target_init") {
310 TargetInitCall = Call;
311 break;
312 }
313 }
314 }
315
316 if (!TargetInitCall)
317 return std::nullopt;
318
319 // Get the kernel mode information from the global variable associated to the
320 // first argument to the call to __kmpc_target_init. Refer to
321 // createTargetInit() to see how this is initialized.
322 Value *InitOperand = TargetInitCall->getArgOperand(i: 0);
323 GlobalVariable *KernelEnv = nullptr;
324 if (auto *Cast = dyn_cast<ConstantExpr>(Val: InitOperand))
325 KernelEnv = cast<GlobalVariable>(Val: Cast->getOperand(i_nocapture: 0));
326 else
327 KernelEnv = cast<GlobalVariable>(Val: InitOperand);
328 auto *KernelEnvInit = cast<ConstantStruct>(Val: KernelEnv->getInitializer());
329 auto *ConfigEnv = cast<ConstantStruct>(Val: KernelEnvInit->getOperand(i_nocapture: 0));
330 auto *KernelMode = cast<ConstantInt>(Val: ConfigEnv->getOperand(i_nocapture: 2));
331 return static_cast<OMPTgtExecModeFlags>(KernelMode->getZExtValue());
332}
333
334static bool isGenericKernel(Function &Fn) {
335 std::optional<omp::OMPTgtExecModeFlags> ExecMode =
336 getTargetKernelExecMode(Kernel&: Fn);
337 return !ExecMode || (*ExecMode & OMP_TGT_EXEC_MODE_GENERIC);
338}
339
340/// Make \p Source branch to \p Target.
341///
342/// Handles two situations:
343/// * \p Source already has an unconditional branch.
344/// * \p Source is a degenerate block (no terminator because the BB is
345/// the current head of the IR construction).
346static void redirectTo(BasicBlock *Source, BasicBlock *Target, DebugLoc DL) {
347 if (Instruction *Term = Source->getTerminatorOrNull()) {
348 auto *Br = cast<UncondBrInst>(Val: Term);
349 BasicBlock *Succ = Br->getSuccessor();
350 Succ->removePredecessor(Pred: Source, /*KeepOneInputPHIs=*/true);
351 Br->setSuccessor(Target);
352 return;
353 }
354
355 auto *NewBr = UncondBrInst::Create(Target, InsertBefore: Source);
356 NewBr->setDebugLoc(DL);
357}
358
359void llvm::spliceBB(IRBuilderBase::InsertPoint IP, BasicBlock *New,
360 bool CreateBranch, DebugLoc DL) {
361 assert(New->getFirstInsertionPt() == New->begin() &&
362 "Target BB must not have PHI nodes");
363
364 // Move instructions to new block.
365 BasicBlock *Old = IP.getBlock();
366 // If the `Old` block is empty then there are no instructions to move. But in
367 // the new debug scheme, it could have trailing debug records which will be
368 // moved to `New` in `spliceDebugInfoEmptyBlock`. We dont want that for 2
369 // reasons:
370 // 1. If `New` is also empty, `BasicBlock::splice` crashes.
371 // 2. Even if `New` is not empty, the rationale to move those records to `New`
372 // (in `spliceDebugInfoEmptyBlock`) does not apply here. That function
373 // assumes that `Old` is optimized out and is going away. This is not the case
374 // here. The `Old` block is still being used e.g. a branch instruction is
375 // added to it later in this function.
376 // So we call `BasicBlock::splice` only when `Old` is not empty.
377 if (!Old->empty())
378 New->splice(ToIt: New->begin(), FromBB: Old, FromBeginIt: IP.getPoint(), FromEndIt: Old->end());
379
380 if (CreateBranch) {
381 auto *NewBr = UncondBrInst::Create(Target: New, InsertBefore: Old);
382 NewBr->setDebugLoc(DL);
383 }
384}
385
386void llvm::spliceBB(IRBuilder<> &Builder, BasicBlock *New, bool CreateBranch) {
387 DebugLoc DebugLoc = Builder.getCurrentDebugLocation();
388 BasicBlock *Old = Builder.GetInsertBlock();
389
390 spliceBB(IP: Builder.saveIP(), New, CreateBranch, DL: DebugLoc);
391 if (CreateBranch)
392 Builder.SetInsertPoint(Old->getTerminator());
393 else
394 Builder.SetInsertPoint(Old);
395
396 // SetInsertPoint also updates the Builder's debug location, but we want to
397 // keep the one the Builder was configured to use.
398 Builder.SetCurrentDebugLocation(DebugLoc);
399}
400
401BasicBlock *llvm::splitBB(IRBuilderBase::InsertPoint IP, bool CreateBranch,
402 DebugLoc DL, llvm::Twine Name) {
403 BasicBlock *Old = IP.getBlock();
404 BasicBlock *New = BasicBlock::Create(
405 Context&: Old->getContext(), Name: Name.isTriviallyEmpty() ? Old->getName() : Name,
406 Parent: Old->getParent(), InsertBefore: Old->getNextNode());
407 spliceBB(IP, New, CreateBranch, DL);
408 New->replaceSuccessorsPhiUsesWith(Old, New);
409 return New;
410}
411
412BasicBlock *llvm::splitBB(IRBuilderBase &Builder, bool CreateBranch,
413 llvm::Twine Name) {
414 DebugLoc DebugLoc = Builder.getCurrentDebugLocation();
415 BasicBlock *New = splitBB(IP: Builder.saveIP(), CreateBranch, DL: DebugLoc, Name);
416 if (CreateBranch)
417 Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator());
418 else
419 Builder.SetInsertPoint(Builder.GetInsertBlock());
420 // SetInsertPoint also updates the Builder's debug location, but we want to
421 // keep the one the Builder was configured to use.
422 Builder.SetCurrentDebugLocation(DebugLoc);
423 return New;
424}
425
426BasicBlock *llvm::splitBB(IRBuilder<> &Builder, bool CreateBranch,
427 llvm::Twine Name) {
428 DebugLoc DebugLoc = Builder.getCurrentDebugLocation();
429 BasicBlock *New = splitBB(IP: Builder.saveIP(), CreateBranch, DL: DebugLoc, Name);
430 if (CreateBranch)
431 Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator());
432 else
433 Builder.SetInsertPoint(Builder.GetInsertBlock());
434 // SetInsertPoint also updates the Builder's debug location, but we want to
435 // keep the one the Builder was configured to use.
436 Builder.SetCurrentDebugLocation(DebugLoc);
437 return New;
438}
439
440BasicBlock *llvm::splitBBWithSuffix(IRBuilderBase &Builder, bool CreateBranch,
441 llvm::Twine Suffix) {
442 BasicBlock *Old = Builder.GetInsertBlock();
443 return splitBB(Builder, CreateBranch, Name: Old->getName() + Suffix);
444}
445
446// This function creates a fake integer value and a fake use for the integer
447// value. It returns the fake value created. This is useful in modeling the
448// extra arguments to the outlined functions.
449Value *createFakeIntVal(IRBuilderBase &Builder,
450 OpenMPIRBuilder::InsertPointTy OuterAllocaIP,
451 llvm::SmallVectorImpl<Instruction *> &ToBeDeleted,
452 OpenMPIRBuilder::InsertPointTy InnerAllocaIP,
453 const Twine &Name = "", bool AsPtr = true,
454 bool Is64Bit = false) {
455 Builder.restoreIP(IP: OuterAllocaIP);
456 IntegerType *IntTy = Is64Bit ? Builder.getInt64Ty() : Builder.getInt32Ty();
457 Instruction *FakeVal;
458 AllocaInst *FakeValAddr =
459 Builder.CreateAlloca(Ty: IntTy, ArraySize: nullptr, Name: Name + ".addr");
460 ToBeDeleted.push_back(Elt: FakeValAddr);
461
462 if (AsPtr) {
463 FakeVal = FakeValAddr;
464 } else {
465 FakeVal = Builder.CreateLoad(Ty: IntTy, Ptr: FakeValAddr, Name: Name + ".val");
466 ToBeDeleted.push_back(Elt: FakeVal);
467 }
468
469 // Generate a fake use of this value
470 Builder.restoreIP(IP: InnerAllocaIP);
471 Instruction *UseFakeVal;
472 if (AsPtr) {
473 UseFakeVal = Builder.CreateLoad(Ty: IntTy, Ptr: FakeVal, Name: Name + ".use");
474 } else {
475 UseFakeVal = cast<BinaryOperator>(Val: Builder.CreateAdd(
476 LHS: FakeVal, RHS: Is64Bit ? Builder.getInt64(C: 10) : Builder.getInt32(C: 10)));
477 }
478 ToBeDeleted.push_back(Elt: UseFakeVal);
479 return FakeVal;
480}
481
482//===----------------------------------------------------------------------===//
483// OpenMPIRBuilderConfig
484//===----------------------------------------------------------------------===//
485
486namespace {
487LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();
488/// Values for bit flags for marking which requires clauses have been used.
489enum OpenMPOffloadingRequiresDirFlags {
490 /// flag undefined.
491 OMP_REQ_UNDEFINED = 0x000,
492 /// no requires directive present.
493 OMP_REQ_NONE = 0x001,
494 /// reverse_offload clause.
495 OMP_REQ_REVERSE_OFFLOAD = 0x002,
496 /// unified_address clause.
497 OMP_REQ_UNIFIED_ADDRESS = 0x004,
498 /// unified_shared_memory clause.
499 OMP_REQ_UNIFIED_SHARED_MEMORY = 0x008,
500 /// dynamic_allocators clause.
501 OMP_REQ_DYNAMIC_ALLOCATORS = 0x010,
502 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_REQ_DYNAMIC_ALLOCATORS)
503};
504
505class OMPCodeExtractor : public CodeExtractor {
506public:
507 OMPCodeExtractor(OpenMPIRBuilder &OMPBuilder, ArrayRef<BasicBlock *> BBs,
508 DominatorTree *DT = nullptr, bool AggregateArgs = false,
509 BlockFrequencyInfo *BFI = nullptr,
510 BranchProbabilityInfo *BPI = nullptr,
511 AssumptionCache *AC = nullptr, bool AllowVarArgs = false,
512 bool AllowAlloca = false,
513 BasicBlock *AllocationBlock = nullptr,
514 ArrayRef<BasicBlock *> DeallocationBlocks = {},
515 std::string Suffix = "", bool ArgsInZeroAddressSpace = false)
516 : CodeExtractor(BBs, DT, AggregateArgs, BFI, BPI, AC, AllowVarArgs,
517 AllowAlloca, AllocationBlock, DeallocationBlocks, Suffix,
518 ArgsInZeroAddressSpace),
519 OMPBuilder(OMPBuilder) {}
520
521 virtual ~OMPCodeExtractor() = default;
522
523protected:
524 OpenMPIRBuilder &OMPBuilder;
525};
526
527class DeviceSharedMemCodeExtractor : public OMPCodeExtractor {
528public:
529 using OMPCodeExtractor::OMPCodeExtractor;
530 virtual ~DeviceSharedMemCodeExtractor() = default;
531
532protected:
533 virtual Instruction *
534 allocateVar(IRBuilder<>::InsertPoint AllocaIP, Type *VarType,
535 const Twine &Name = Twine(""),
536 AddrSpaceCastInst **CastedAlloc = nullptr) override {
537 return OMPBuilder.createOMPAllocShared(Loc: AllocaIP, VarType, Name);
538 }
539
540 virtual Instruction *deallocateVar(IRBuilder<>::InsertPoint DeallocIP,
541 Value *Var, Type *VarType) override {
542 return OMPBuilder.createOMPFreeShared(Loc: DeallocIP, Addr: Var, VarType);
543 }
544};
545
546/// Helper storing information about regions to outline using device shared
547/// memory for intermediate allocations.
548struct DeviceSharedMemOutlineInfo : public OpenMPIRBuilder::OutlineInfo {
549 OpenMPIRBuilder &OMPBuilder;
550
551 DeviceSharedMemOutlineInfo(OpenMPIRBuilder &OMPBuilder)
552 : OMPBuilder(OMPBuilder) {}
553 virtual ~DeviceSharedMemOutlineInfo() = default;
554
555 virtual std::unique_ptr<CodeExtractor>
556 createCodeExtractor(ArrayRef<BasicBlock *> Blocks,
557 bool ArgsInZeroAddressSpace,
558 Twine Suffix = Twine("")) override;
559};
560
561} // anonymous namespace
562
563OpenMPIRBuilderConfig::OpenMPIRBuilderConfig()
564 : RequiresFlags(OMP_REQ_UNDEFINED) {}
565
566OpenMPIRBuilderConfig::OpenMPIRBuilderConfig(
567 bool IsTargetDevice, bool IsGPU, bool OpenMPOffloadMandatory,
568 bool HasRequiresReverseOffload, bool HasRequiresUnifiedAddress,
569 bool HasRequiresUnifiedSharedMemory, bool HasRequiresDynamicAllocators)
570 : IsTargetDevice(IsTargetDevice), IsGPU(IsGPU),
571 OpenMPOffloadMandatory(OpenMPOffloadMandatory),
572 RequiresFlags(OMP_REQ_UNDEFINED) {
573 if (HasRequiresReverseOffload)
574 RequiresFlags |= OMP_REQ_REVERSE_OFFLOAD;
575 if (HasRequiresUnifiedAddress)
576 RequiresFlags |= OMP_REQ_UNIFIED_ADDRESS;
577 if (HasRequiresUnifiedSharedMemory)
578 RequiresFlags |= OMP_REQ_UNIFIED_SHARED_MEMORY;
579 if (HasRequiresDynamicAllocators)
580 RequiresFlags |= OMP_REQ_DYNAMIC_ALLOCATORS;
581}
582
583bool OpenMPIRBuilderConfig::hasRequiresReverseOffload() const {
584 return RequiresFlags & OMP_REQ_REVERSE_OFFLOAD;
585}
586
587bool OpenMPIRBuilderConfig::hasRequiresUnifiedAddress() const {
588 return RequiresFlags & OMP_REQ_UNIFIED_ADDRESS;
589}
590
591bool OpenMPIRBuilderConfig::hasRequiresUnifiedSharedMemory() const {
592 return RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY;
593}
594
595bool OpenMPIRBuilderConfig::hasRequiresDynamicAllocators() const {
596 return RequiresFlags & OMP_REQ_DYNAMIC_ALLOCATORS;
597}
598
599int64_t OpenMPIRBuilderConfig::getRequiresFlags() const {
600 return hasRequiresFlags() ? RequiresFlags
601 : static_cast<int64_t>(OMP_REQ_NONE);
602}
603
604void OpenMPIRBuilderConfig::setHasRequiresReverseOffload(bool Value) {
605 if (Value)
606 RequiresFlags |= OMP_REQ_REVERSE_OFFLOAD;
607 else
608 RequiresFlags &= ~OMP_REQ_REVERSE_OFFLOAD;
609}
610
611void OpenMPIRBuilderConfig::setHasRequiresUnifiedAddress(bool Value) {
612 if (Value)
613 RequiresFlags |= OMP_REQ_UNIFIED_ADDRESS;
614 else
615 RequiresFlags &= ~OMP_REQ_UNIFIED_ADDRESS;
616}
617
618void OpenMPIRBuilderConfig::setHasRequiresUnifiedSharedMemory(bool Value) {
619 if (Value)
620 RequiresFlags |= OMP_REQ_UNIFIED_SHARED_MEMORY;
621 else
622 RequiresFlags &= ~OMP_REQ_UNIFIED_SHARED_MEMORY;
623}
624
625void OpenMPIRBuilderConfig::setHasRequiresDynamicAllocators(bool Value) {
626 if (Value)
627 RequiresFlags |= OMP_REQ_DYNAMIC_ALLOCATORS;
628 else
629 RequiresFlags &= ~OMP_REQ_DYNAMIC_ALLOCATORS;
630}
631
632//===----------------------------------------------------------------------===//
633// OpenMPIRBuilder
634//===----------------------------------------------------------------------===//
635
636void OpenMPIRBuilder::getKernelArgsVector(TargetKernelArgs &KernelArgs,
637 IRBuilderBase &Builder,
638 SmallVector<Value *> &ArgsVector) {
639 Value *Version = Builder.getInt32(OMP_KERNEL_ARG_VERSION);
640 Value *PointerNum = Builder.getInt32(C: KernelArgs.NumTargetItems);
641 auto Int32Ty = Type::getInt32Ty(C&: Builder.getContext());
642 constexpr size_t MaxDim = 3;
643 Value *ZeroArray = Constant::getNullValue(Ty: ArrayType::get(ElementType: Int32Ty, NumElements: MaxDim));
644
645 Value *HasNoWaitFlag = Builder.getInt64(C: KernelArgs.HasNoWait);
646
647 Value *DynCGroupMemFallbackFlag =
648 Builder.getInt64(C: static_cast<uint64_t>(KernelArgs.DynCGroupMemFallback));
649 DynCGroupMemFallbackFlag = Builder.CreateShl(LHS: DynCGroupMemFallbackFlag, RHS: 2);
650
651 Value *StrictFlag = Builder.getInt64(C: KernelArgs.StrictBlocksAndThreads);
652 StrictFlag = Builder.CreateShl(LHS: StrictFlag, RHS: 6);
653
654 Value *Flags = Builder.CreateOr(LHS: HasNoWaitFlag, RHS: DynCGroupMemFallbackFlag);
655 Flags = Builder.CreateOr(LHS: Flags, RHS: StrictFlag);
656
657 assert(!KernelArgs.NumTeams.empty() && !KernelArgs.NumThreads.empty());
658
659 Value *NumTeams3D =
660 Builder.CreateInsertValue(Agg: ZeroArray, Val: KernelArgs.NumTeams[0], Idxs: {0});
661 Value *NumThreads3D =
662 Builder.CreateInsertValue(Agg: ZeroArray, Val: KernelArgs.NumThreads[0], Idxs: {0});
663 for (unsigned I :
664 seq<unsigned>(Begin: 1, End: std::min(a: KernelArgs.NumTeams.size(), b: MaxDim)))
665 NumTeams3D =
666 Builder.CreateInsertValue(Agg: NumTeams3D, Val: KernelArgs.NumTeams[I], Idxs: {I});
667 for (unsigned I :
668 seq<unsigned>(Begin: 1, End: std::min(a: KernelArgs.NumThreads.size(), b: MaxDim)))
669 NumThreads3D =
670 Builder.CreateInsertValue(Agg: NumThreads3D, Val: KernelArgs.NumThreads[I], Idxs: {I});
671
672 ArgsVector = {Version,
673 PointerNum,
674 KernelArgs.RTArgs.BasePointersArray,
675 KernelArgs.RTArgs.PointersArray,
676 KernelArgs.RTArgs.SizesArray,
677 KernelArgs.RTArgs.MapTypesArray,
678 KernelArgs.RTArgs.MapNamesArray,
679 KernelArgs.RTArgs.MappersArray,
680 KernelArgs.NumIterations,
681 Flags,
682 NumTeams3D,
683 NumThreads3D,
684 KernelArgs.DynCGroupMem};
685}
686
687void OpenMPIRBuilder::addAttributes(omp::RuntimeFunction FnID, Function &Fn) {
688 LLVMContext &Ctx = Fn.getContext();
689
690 // Get the function's current attributes.
691 auto Attrs = Fn.getAttributes();
692 auto FnAttrs = Attrs.getFnAttrs();
693 auto RetAttrs = Attrs.getRetAttrs();
694 SmallVector<AttributeSet, 4> ArgAttrs;
695 for (size_t ArgNo = 0; ArgNo < Fn.arg_size(); ++ArgNo)
696 ArgAttrs.emplace_back(Args: Attrs.getParamAttrs(ArgNo));
697
698 // Add AS to FnAS while taking special care with integer extensions.
699 auto addAttrSet = [&](AttributeSet &FnAS, const AttributeSet &AS,
700 bool Param = true) -> void {
701 bool HasSignExt = AS.hasAttribute(Kind: Attribute::SExt);
702 bool HasZeroExt = AS.hasAttribute(Kind: Attribute::ZExt);
703 if (HasSignExt || HasZeroExt) {
704 assert(AS.getNumAttributes() == 1 &&
705 "Currently not handling extension attr combined with others.");
706 if (Param) {
707 if (auto AK = TargetLibraryInfo::getExtAttrForI32Param(T, Signed: HasSignExt))
708 FnAS = FnAS.addAttribute(C&: Ctx, Kind: AK);
709 } else if (auto AK =
710 TargetLibraryInfo::getExtAttrForI32Return(T, Signed: HasSignExt))
711 FnAS = FnAS.addAttribute(C&: Ctx, Kind: AK);
712 } else {
713 FnAS = FnAS.addAttributes(C&: Ctx, AS);
714 }
715 };
716
717#define OMP_ATTRS_SET(VarName, AttrSet) AttributeSet VarName = AttrSet;
718#include "llvm/Frontend/OpenMP/OMPKinds.def"
719
720 // Add attributes to the function declaration.
721 switch (FnID) {
722#define OMP_RTL_ATTRS(Enum, FnAttrSet, RetAttrSet, ArgAttrSets) \
723 case Enum: \
724 FnAttrs = FnAttrs.addAttributes(Ctx, FnAttrSet); \
725 addAttrSet(RetAttrs, RetAttrSet, /*Param*/ false); \
726 for (size_t ArgNo = 0; ArgNo < ArgAttrSets.size(); ++ArgNo) \
727 addAttrSet(ArgAttrs[ArgNo], ArgAttrSets[ArgNo]); \
728 Fn.setAttributes(AttributeList::get(Ctx, FnAttrs, RetAttrs, ArgAttrs)); \
729 break;
730#include "llvm/Frontend/OpenMP/OMPKinds.def"
731 default:
732 // Attributes are optional.
733 break;
734 }
735}
736
737FunctionCallee
738OpenMPIRBuilder::getOrCreateRuntimeFunction(Module &M, RuntimeFunction FnID) {
739 FunctionType *FnTy = nullptr;
740 Function *Fn = nullptr;
741
742 // Try to find the declation in the module first.
743 switch (FnID) {
744#define OMP_RTL(Enum, Str, IsVarArg, ReturnType, ...) \
745 case Enum: \
746 FnTy = FunctionType::get(ReturnType, ArrayRef<Type *>{__VA_ARGS__}, \
747 IsVarArg); \
748 Fn = M.getFunction(Str); \
749 break;
750#include "llvm/Frontend/OpenMP/OMPKinds.def"
751 }
752
753 if (!Fn) {
754 // Create a new declaration if we need one.
755 switch (FnID) {
756#define OMP_RTL(Enum, Str, ...) \
757 case Enum: \
758 Fn = Function::Create(FnTy, GlobalValue::ExternalLinkage, Str, M); \
759 break;
760#include "llvm/Frontend/OpenMP/OMPKinds.def"
761 }
762 Fn->setCallingConv(Config.getRuntimeCC());
763 // Add information if the runtime function takes a callback function
764 if (FnID == OMPRTL___kmpc_fork_call || FnID == OMPRTL___kmpc_fork_teams) {
765 if (!Fn->hasMetadata(KindID: LLVMContext::MD_callback)) {
766 LLVMContext &Ctx = Fn->getContext();
767 MDBuilder MDB(Ctx);
768 // Annotate the callback behavior of the runtime function:
769 // - The callback callee is argument number 2 (microtask).
770 // - The first two arguments of the callback callee are unknown (-1).
771 // - All variadic arguments to the runtime function are passed to the
772 // callback callee.
773 Fn->addMetadata(
774 KindID: LLVMContext::MD_callback,
775 MD&: *MDNode::get(Context&: Ctx, MDs: {MDB.createCallbackEncoding(
776 CalleeArgNo: 2, Arguments: {-1, -1}, /* VarArgsArePassed */ true)}));
777 }
778 }
779
780 LLVM_DEBUG(dbgs() << "Created OpenMP runtime function " << Fn->getName()
781 << " with type " << *Fn->getFunctionType() << "\n");
782 addAttributes(FnID, Fn&: *Fn);
783
784 } else {
785 LLVM_DEBUG(dbgs() << "Found OpenMP runtime function " << Fn->getName()
786 << " with type " << *Fn->getFunctionType() << "\n");
787 }
788
789 assert(Fn && "Failed to create OpenMP runtime function");
790
791 return {FnTy, Fn};
792}
793
794Expected<BasicBlock *>
795OpenMPIRBuilder::FinalizationInfo::getFiniBB(IRBuilderBase &Builder) {
796 if (!FiniBB) {
797 Function *ParentFunc = Builder.GetInsertBlock()->getParent();
798 IRBuilderBase::InsertPointGuard Guard(Builder);
799 FiniBB = BasicBlock::Create(Context&: Builder.getContext(), Name: ".fini", Parent: ParentFunc);
800 Builder.SetInsertPoint(FiniBB);
801 // FiniCB adds the branch to the exit stub.
802 if (Error Err = FiniCB(Builder.saveIP()))
803 return Err;
804 }
805 return FiniBB;
806}
807
808Error OpenMPIRBuilder::FinalizationInfo::mergeFiniBB(IRBuilderBase &Builder,
809 BasicBlock *OtherFiniBB) {
810 // Simple case: FiniBB does not exist yet: re-use OtherFiniBB.
811 if (!FiniBB) {
812 FiniBB = OtherFiniBB;
813
814 Builder.SetInsertPoint(FiniBB->getFirstNonPHIIt());
815 if (Error Err = FiniCB(Builder.saveIP()))
816 return Err;
817
818 return Error::success();
819 }
820
821 // Move instructions from FiniBB to the start of OtherFiniBB.
822 auto EndIt = FiniBB->end();
823 if (FiniBB->size() >= 1)
824 if (auto Prev = std::prev(x: EndIt); Prev->isTerminator())
825 EndIt = Prev;
826 OtherFiniBB->splice(ToIt: OtherFiniBB->getFirstNonPHIIt(), FromBB: FiniBB, FromBeginIt: FiniBB->begin(),
827 FromEndIt: EndIt);
828
829 FiniBB->replaceAllUsesWith(V: OtherFiniBB);
830 FiniBB->eraseFromParent();
831 FiniBB = OtherFiniBB;
832 return Error::success();
833}
834
835Function *OpenMPIRBuilder::getOrCreateRuntimeFunctionPtr(RuntimeFunction FnID) {
836 FunctionCallee RTLFn = getOrCreateRuntimeFunction(M, FnID);
837 auto *Fn = dyn_cast<llvm::Function>(Val: RTLFn.getCallee());
838 assert(Fn && "Failed to create OpenMP runtime function pointer");
839 return Fn;
840}
841
842CallInst *OpenMPIRBuilder::createRuntimeFunctionCall(FunctionCallee Callee,
843 ArrayRef<Value *> Args,
844 StringRef Name) {
845 CallInst *Call = Builder.CreateCall(Callee, Args, Name);
846 Call->setCallingConv(Config.getRuntimeCC());
847 return Call;
848}
849
850void OpenMPIRBuilder::initialize() { initializeTypes(M); }
851
852static void raiseUserConstantDataAllocasToEntryBlock(IRBuilderBase &Builder,
853 Function *Function) {
854 BasicBlock &EntryBlock = Function->getEntryBlock();
855 BasicBlock::iterator MoveLocInst = EntryBlock.getFirstNonPHIIt();
856
857 // Loop over blocks looking for constant allocas, skipping the entry block
858 // as any allocas there are already in the desired location.
859 for (auto Block = std::next(x: Function->begin(), n: 1); Block != Function->end();
860 Block++) {
861 for (auto Inst = Block->getReverseIterator()->begin();
862 Inst != Block->getReverseIterator()->end();) {
863 if (auto *AllocaInst = dyn_cast_if_present<llvm::AllocaInst>(Val&: Inst)) {
864 Inst++;
865 if (!isa<ConstantData>(Val: AllocaInst->getArraySize()))
866 continue;
867 AllocaInst->moveBeforePreserving(MovePos: MoveLocInst);
868 } else {
869 Inst++;
870 }
871 }
872 }
873}
874
875static void hoistNonEntryAllocasToEntryBlock(llvm::BasicBlock &Block) {
876 llvm::SmallVector<llvm::Instruction *> AllocasToMove;
877
878 auto ShouldHoistAlloca = [](const llvm::AllocaInst &AllocaInst) {
879 // TODO: For now, we support simple static allocations, we might need to
880 // move non-static ones as well. However, this will need further analysis to
881 // move the lenght arguments as well.
882 return !AllocaInst.isArrayAllocation();
883 };
884
885 for (llvm::Instruction &Inst : Block)
886 if (auto *AllocaInst = llvm::dyn_cast<llvm::AllocaInst>(Val: &Inst))
887 if (ShouldHoistAlloca(*AllocaInst))
888 AllocasToMove.push_back(Elt: AllocaInst);
889
890 auto InsertPoint =
891 Block.getParent()->getEntryBlock().getTerminator()->getIterator();
892
893 for (llvm::Instruction *AllocaInst : AllocasToMove)
894 AllocaInst->moveBefore(InsertPos: InsertPoint);
895}
896
897static void hoistNonEntryAllocasToEntryBlock(llvm::Function *Func) {
898 PostDominatorTree PostDomTree(*Func);
899 for (llvm::BasicBlock &BB : *Func)
900 if (PostDomTree.properlyDominates(A: &BB, B: &Func->getEntryBlock()))
901 hoistNonEntryAllocasToEntryBlock(Block&: BB);
902}
903
904void OpenMPIRBuilder::finalize(Function *Fn) {
905 SmallPtrSet<BasicBlock *, 32> ParallelRegionBlockSet;
906 SmallVector<BasicBlock *, 32> Blocks;
907 SmallVector<std::unique_ptr<OutlineInfo>, 16> DeferredOutlines;
908 for (std::unique_ptr<OutlineInfo> &OI : OutlineInfos) {
909 // Skip functions that have not finalized yet; may happen with nested
910 // function generation.
911 if (Fn && OI->getFunction() != Fn) {
912 DeferredOutlines.push_back(Elt: std::move(OI));
913 continue;
914 }
915
916 ParallelRegionBlockSet.clear();
917 Blocks.clear();
918 OI->collectBlocks(BlockSet&: ParallelRegionBlockSet, BlockVector&: Blocks);
919
920 Function *OuterFn = OI->getFunction();
921 CodeExtractorAnalysisCache CEAC(*OuterFn);
922 // If we generate code for the target device, we need to allocate
923 // struct for aggregate params in the device default alloca address space.
924 // OpenMP runtime requires that the params of the extracted functions are
925 // passed as zero address space pointers. This flag ensures that
926 // CodeExtractor generates correct code for extracted functions
927 // which are used by OpenMP runtime.
928 bool ArgsInZeroAddressSpace = Config.isTargetDevice();
929 std::unique_ptr<CodeExtractor> Extractor =
930 OI->createCodeExtractor(Blocks, ArgsInZeroAddressSpace, Suffix: ".omp_par");
931
932 LLVM_DEBUG(dbgs() << "Before outlining: " << *OuterFn << "\n");
933 LLVM_DEBUG(dbgs() << "Entry " << OI->EntryBB->getName()
934 << " Exit: " << OI->ExitBB->getName() << "\n");
935 assert(Extractor->isEligible() &&
936 "Expected OpenMP outlining to be possible!");
937
938 for (auto *V : OI->ExcludeArgsFromAggregate)
939 Extractor->excludeArgFromAggregate(Arg: V);
940
941 Function *OutlinedFn =
942 Extractor->extractCodeRegion(CEAC, Inputs&: OI->Inputs, Outputs&: OI->Outputs);
943
944 // Forward target-cpu, target-features attributes to the outlined function.
945 auto TargetCpuAttr = OuterFn->getFnAttribute(Kind: "target-cpu");
946 if (TargetCpuAttr.isStringAttribute())
947 OutlinedFn->addFnAttr(Attr: TargetCpuAttr);
948
949 auto TargetFeaturesAttr = OuterFn->getFnAttribute(Kind: "target-features");
950 if (TargetFeaturesAttr.isStringAttribute())
951 OutlinedFn->addFnAttr(Attr: TargetFeaturesAttr);
952
953 LLVM_DEBUG(dbgs() << "After outlining: " << *OuterFn << "\n");
954 LLVM_DEBUG(dbgs() << " Outlined function: " << *OutlinedFn << "\n");
955 assert(OutlinedFn->getReturnType()->isVoidTy() &&
956 "OpenMP outlined functions should not return a value!");
957
958 // For compability with the clang CG we move the outlined function after the
959 // one with the parallel region.
960 OutlinedFn->removeFromParent();
961 M.getFunctionList().insertAfter(where: OuterFn->getIterator(), New: OutlinedFn);
962
963 // Remove the artificial entry introduced by the extractor right away, we
964 // made our own entry block after all.
965 {
966 BasicBlock &ArtificialEntry = OutlinedFn->getEntryBlock();
967 assert(ArtificialEntry.getUniqueSuccessor() == OI->EntryBB);
968 assert(OI->EntryBB->getUniquePredecessor() == &ArtificialEntry);
969 // Move instructions from the to-be-deleted ArtificialEntry to the entry
970 // basic block of the parallel region. CodeExtractor generates
971 // instructions to unwrap the aggregate argument and may sink
972 // allocas/bitcasts for values that are solely used in the outlined region
973 // and do not escape.
974 assert(!ArtificialEntry.empty() &&
975 "Expected instructions to add in the outlined region entry");
976 for (BasicBlock::reverse_iterator It = ArtificialEntry.rbegin(),
977 End = ArtificialEntry.rend();
978 It != End;) {
979 Instruction &I = *It;
980 It++;
981
982 if (I.isTerminator()) {
983 // Absorb any debug value that terminator may have
984 if (Instruction *TI = OI->EntryBB->getTerminatorOrNull())
985 TI->adoptDbgRecords(BB: &ArtificialEntry, It: I.getIterator(), InsertAtHead: false);
986 continue;
987 }
988
989 I.moveBeforePreserving(BB&: *OI->EntryBB,
990 I: OI->EntryBB->getFirstInsertionPt());
991 }
992
993 OI->EntryBB->moveBefore(MovePos: &ArtificialEntry);
994 ArtificialEntry.eraseFromParent();
995 }
996 assert(&OutlinedFn->getEntryBlock() == OI->EntryBB);
997 assert(OutlinedFn && OutlinedFn->hasNUses(1));
998
999 // Run a user callback, e.g. to add attributes.
1000 if (OI->PostOutlineCB)
1001 OI->PostOutlineCB(*OutlinedFn);
1002
1003 if (OI->FixUpNonEntryAllocas)
1004 hoistNonEntryAllocasToEntryBlock(Func: OutlinedFn);
1005 }
1006
1007 // Remove work items that have been completed.
1008 OutlineInfos = std::move(DeferredOutlines);
1009
1010 // The createTarget functions embeds user written code into
1011 // the target region which may inject allocas which need to
1012 // be moved to the entry block of our target or risk malformed
1013 // optimisations by later passes, this is only relevant for
1014 // the device pass which appears to be a little more delicate
1015 // when it comes to optimisations (however, we do not block on
1016 // that here, it's up to the inserter to the list to do so).
1017 // This notbaly has to occur after the OutlinedInfo candidates
1018 // have been extracted so we have an end product that will not
1019 // be implicitly adversely affected by any raises unless
1020 // intentionally appended to the list.
1021 // NOTE: This only does so for ConstantData, it could be extended
1022 // to ConstantExpr's with further effort, however, they should
1023 // largely be folded when they get here. Extending it to runtime
1024 // defined/read+writeable allocation sizes would be non-trivial
1025 // (need to factor in movement of any stores to variables the
1026 // allocation size depends on, as well as the usual loads,
1027 // otherwise it'll yield the wrong result after movement) and
1028 // likely be more suitable as an LLVM optimisation pass.
1029 for (Function *F : ConstantAllocaRaiseCandidates)
1030 raiseUserConstantDataAllocasToEntryBlock(Builder, Function: F);
1031
1032 EmitMetadataErrorReportFunctionTy &&ErrorReportFn =
1033 [](EmitMetadataErrorKind Kind,
1034 const TargetRegionEntryInfo &EntryInfo) -> void {
1035 errs() << "Error of kind: " << Kind
1036 << " when emitting offload entries and metadata during "
1037 "OMPIRBuilder finalization \n";
1038 };
1039
1040 if (!OffloadInfoManager.empty())
1041 createOffloadEntriesAndInfoMetadata(ErrorReportFunction&: ErrorReportFn);
1042
1043 if (Config.EmitLLVMUsedMetaInfo.value_or(u: false)) {
1044 std::vector<WeakTrackingVH> LLVMCompilerUsed = {
1045 M.getGlobalVariable(Name: "__openmp_nvptx_data_transfer_temporary_storage")};
1046 emitUsed(Name: "llvm.compiler.used", List: LLVMCompilerUsed);
1047 }
1048
1049 IsFinalized = true;
1050}
1051
1052bool OpenMPIRBuilder::isFinalized() { return IsFinalized; }
1053
1054OpenMPIRBuilder::~OpenMPIRBuilder() {
1055 assert(OutlineInfos.empty() && "There must be no outstanding outlinings");
1056}
1057
1058GlobalValue *OpenMPIRBuilder::createGlobalFlag(unsigned Value, StringRef Name) {
1059 IntegerType *I32Ty = Type::getInt32Ty(C&: M.getContext());
1060 auto *GV =
1061 new GlobalVariable(M, I32Ty,
1062 /* isConstant = */ true, GlobalValue::WeakODRLinkage,
1063 ConstantInt::get(Ty: I32Ty, V: Value), Name);
1064 GV->setVisibility(GlobalValue::HiddenVisibility);
1065
1066 return GV;
1067}
1068
1069void OpenMPIRBuilder::emitUsed(StringRef Name, ArrayRef<WeakTrackingVH> List) {
1070 if (List.empty())
1071 return;
1072
1073 // Convert List to what ConstantArray needs.
1074 SmallVector<Constant *, 8> UsedArray;
1075 UsedArray.resize(N: List.size());
1076 for (unsigned I = 0, E = List.size(); I != E; ++I)
1077 UsedArray[I] = ConstantExpr::getPointerBitCastOrAddrSpaceCast(
1078 C: cast<Constant>(Val: &*List[I]), Ty: Builder.getPtrTy());
1079
1080 if (UsedArray.empty())
1081 return;
1082 ArrayType *ATy = ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: UsedArray.size());
1083
1084 auto *GV = new GlobalVariable(M, ATy, false, GlobalValue::AppendingLinkage,
1085 ConstantArray::get(T: ATy, V: UsedArray), Name);
1086
1087 GV->setSection("llvm.metadata");
1088}
1089
1090GlobalVariable *
1091OpenMPIRBuilder::emitKernelExecutionMode(StringRef KernelName,
1092 OMPTgtExecModeFlags Mode) {
1093 auto *Int8Ty = Builder.getInt8Ty();
1094 auto *GVMode = new GlobalVariable(
1095 M, Int8Ty, /*isConstant=*/true, GlobalValue::WeakAnyLinkage,
1096 ConstantInt::get(Ty: Int8Ty, V: Mode), Twine(KernelName, "_exec_mode"));
1097 GVMode->setVisibility(GlobalVariable::ProtectedVisibility);
1098 return GVMode;
1099}
1100
1101Constant *OpenMPIRBuilder::getOrCreateIdent(Constant *SrcLocStr,
1102 uint32_t SrcLocStrSize,
1103 IdentFlag LocFlags,
1104 unsigned Reserve2Flags) {
1105 // Enable "C-mode".
1106 LocFlags |= OMP_IDENT_FLAG_KMPC;
1107
1108 Constant *&Ident =
1109 IdentMap[{SrcLocStr, uint64_t(LocFlags) << 31 | Reserve2Flags}];
1110 if (!Ident) {
1111 Constant *I32Null = ConstantInt::getNullValue(Ty: Int32);
1112 Constant *IdentData[] = {I32Null,
1113 ConstantInt::get(Ty: Int32, V: uint32_t(LocFlags)),
1114 ConstantInt::get(Ty: Int32, V: Reserve2Flags),
1115 ConstantInt::get(Ty: Int32, V: SrcLocStrSize), SrcLocStr};
1116
1117 size_t SrcLocStrArgIdx = 4;
1118 if (OpenMPIRBuilder::Ident->getElementType(N: SrcLocStrArgIdx)
1119 ->getPointerAddressSpace() !=
1120 IdentData[SrcLocStrArgIdx]->getType()->getPointerAddressSpace())
1121 IdentData[SrcLocStrArgIdx] = ConstantExpr::getAddrSpaceCast(
1122 C: SrcLocStr, Ty: OpenMPIRBuilder::Ident->getElementType(N: SrcLocStrArgIdx));
1123 Constant *Initializer =
1124 ConstantStruct::get(T: OpenMPIRBuilder::Ident, V: IdentData);
1125
1126 // Look for existing encoding of the location + flags, not needed but
1127 // minimizes the difference to the existing solution while we transition.
1128 for (GlobalVariable &GV : M.globals())
1129 if (GV.getValueType() == OpenMPIRBuilder::Ident && GV.hasInitializer())
1130 if (GV.getInitializer() == Initializer)
1131 Ident = &GV;
1132
1133 if (!Ident) {
1134 auto *GV = new GlobalVariable(
1135 M, OpenMPIRBuilder::Ident,
1136 /* isConstant = */ true, GlobalValue::PrivateLinkage, Initializer, "",
1137 nullptr, GlobalValue::NotThreadLocal,
1138 M.getDataLayout().getDefaultGlobalsAddressSpace());
1139 GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
1140 GV->setAlignment(Align(8));
1141 Ident = GV;
1142 }
1143 }
1144
1145 return ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: Ident, Ty: IdentPtr);
1146}
1147
1148Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(StringRef LocStr,
1149 uint32_t &SrcLocStrSize) {
1150 SrcLocStrSize = LocStr.size();
1151 Constant *&SrcLocStr = SrcLocStrMap[LocStr];
1152 if (!SrcLocStr) {
1153 Constant *Initializer =
1154 ConstantDataArray::getString(Context&: M.getContext(), Initializer: LocStr);
1155
1156 // Look for existing encoding of the location, not needed but minimizes the
1157 // difference to the existing solution while we transition.
1158 for (GlobalVariable &GV : M.globals())
1159 if (GV.isConstant() && GV.hasInitializer() &&
1160 GV.getInitializer() == Initializer)
1161 return SrcLocStr = ConstantExpr::getPointerCast(C: &GV, Ty: Int8Ptr);
1162
1163 SrcLocStr = Builder.CreateGlobalString(
1164 Str: LocStr, /*Name=*/"", AddressSpace: M.getDataLayout().getDefaultGlobalsAddressSpace(),
1165 M: &M);
1166 }
1167 return SrcLocStr;
1168}
1169
1170Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(StringRef FunctionName,
1171 StringRef FileName,
1172 unsigned Line, unsigned Column,
1173 uint32_t &SrcLocStrSize) {
1174 SmallString<128> Buffer;
1175 Buffer.push_back(Elt: ';');
1176 Buffer.append(RHS: FileName);
1177 Buffer.push_back(Elt: ';');
1178 Buffer.append(RHS: FunctionName);
1179 Buffer.push_back(Elt: ';');
1180 Buffer.append(RHS: std::to_string(val: Line));
1181 Buffer.push_back(Elt: ';');
1182 Buffer.append(RHS: std::to_string(val: Column));
1183 Buffer.push_back(Elt: ';');
1184 Buffer.push_back(Elt: ';');
1185 return getOrCreateSrcLocStr(LocStr: Buffer.str(), SrcLocStrSize);
1186}
1187
1188Constant *
1189OpenMPIRBuilder::getOrCreateDefaultSrcLocStr(uint32_t &SrcLocStrSize) {
1190 StringRef UnknownLoc = ";unknown;unknown;0;0;;";
1191 return getOrCreateSrcLocStr(LocStr: UnknownLoc, SrcLocStrSize);
1192}
1193
1194Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(DebugLoc DL,
1195 uint32_t &SrcLocStrSize,
1196 Function *F) {
1197 DILocation *DIL = DL.get();
1198 if (!DIL)
1199 return getOrCreateDefaultSrcLocStr(SrcLocStrSize);
1200 StringRef FileName =
1201 !DIL->getFilename().empty() ? DIL->getFilename() : M.getName();
1202 StringRef Function = DIL->getScope()->getSubprogram()->getName();
1203 if (Function.empty() && F)
1204 Function = F->getName();
1205 return getOrCreateSrcLocStr(FunctionName: Function, FileName, Line: DIL->getLine(),
1206 Column: DIL->getColumn(), SrcLocStrSize);
1207}
1208
1209Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(const LocationDescription &Loc,
1210 uint32_t &SrcLocStrSize) {
1211 return getOrCreateSrcLocStr(DL: Loc.DL, SrcLocStrSize,
1212 F: Loc.IP.getBlock()->getParent());
1213}
1214
1215Value *OpenMPIRBuilder::getOrCreateThreadID(Value *Ident) {
1216 return createRuntimeFunctionCall(
1217 Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_global_thread_num), Args: Ident,
1218 Name: "omp_global_thread_num");
1219}
1220
1221OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTargetInReduction(
1222 const LocationDescription &Loc, ArrayRef<Value *> OrigPtrs,
1223 ArrayRef<Type *> ResultPtrTys,
1224 function_ref<void(unsigned, Value *)> MapPrivateCB) {
1225 assert(OrigPtrs.size() == ResultPtrTys.size() &&
1226 "expected one result pointer type per in_reduction item");
1227 if (!updateToLocation(Loc))
1228 return Loc.IP;
1229 if (OrigPtrs.empty())
1230 return Builder.saveIP();
1231
1232 // Compute the executing thread's gtid once for the whole target body and
1233 // reuse it for every in_reduction lookup, so a target with several
1234 // in_reduction items does not emit a redundant __kmpc_global_thread_num per
1235 // item.
1236 uint32_t SrcLocStrSize;
1237 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1238 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
1239 Value *Gtid = getOrCreateThreadID(Ident);
1240
1241 // The runtime entry point takes (and returns) a generic, default-address-
1242 // space `ptr`. A NULL descriptor makes the runtime walk the enclosing
1243 // taskgroups to find the matching task_reduction registration for the item.
1244 Type *PtrTy = PointerType::getUnqual(C&: M.getContext());
1245 Value *NullDesc = ConstantPointerNull::get(T: PtrTy);
1246 FunctionCallee GetThData =
1247 getOrCreateRuntimeFunction(M, FnID: OMPRTL___kmpc_task_reduction_get_th_data);
1248
1249 for (unsigned Idx = 0; Idx < OrigPtrs.size(); ++Idx) {
1250 // Normalize a non-default-address-space original pointer to the generic
1251 // address space before the call.
1252 Value *OrigPtr = OrigPtrs[Idx];
1253 if (auto *OrigPtrTy = dyn_cast<PointerType>(Val: OrigPtr->getType());
1254 OrigPtrTy && OrigPtrTy->getAddressSpace() != 0)
1255 OrigPtr = Builder.CreateAddrSpaceCast(V: OrigPtr, DestTy: PtrTy);
1256
1257 Value *Priv = Builder.CreateCall(Callee: GetThData, Args: {Gtid, NullDesc, OrigPtr},
1258 Name: "omp.inred.priv");
1259
1260 // Cast the returned private pointer back to the requested address space
1261 // when it differs.
1262 if (auto *ResPtrTy = dyn_cast<PointerType>(Val: ResultPtrTys[Idx]);
1263 ResPtrTy && ResPtrTy->getAddressSpace() != 0)
1264 Priv = Builder.CreateAddrSpaceCast(V: Priv, DestTy: ResultPtrTys[Idx]);
1265
1266 MapPrivateCB(Idx, Priv);
1267 }
1268 return Builder.saveIP();
1269}
1270
1271OpenMPIRBuilder::InsertPointOrErrorTy
1272OpenMPIRBuilder::createBarrier(const LocationDescription &Loc, Directive Kind,
1273 bool ForceSimpleCall, bool CheckCancelFlag) {
1274 if (!updateToLocation(Loc))
1275 return Loc.IP;
1276
1277 // Build call __kmpc_cancel_barrier(loc, thread_id) or
1278 // __kmpc_barrier(loc, thread_id);
1279
1280 IdentFlag BarrierLocFlags;
1281 switch (Kind) {
1282 case OMPD_for:
1283 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_FOR;
1284 break;
1285 case OMPD_sections:
1286 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_SECTIONS;
1287 break;
1288 case OMPD_single:
1289 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_SINGLE;
1290 break;
1291 case OMPD_barrier:
1292 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_EXPL;
1293 break;
1294 default:
1295 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL;
1296 break;
1297 }
1298
1299 uint32_t SrcLocStrSize;
1300 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1301 Value *Args[] = {
1302 getOrCreateIdent(SrcLocStr, SrcLocStrSize, LocFlags: BarrierLocFlags),
1303 getOrCreateThreadID(Ident: getOrCreateIdent(SrcLocStr, SrcLocStrSize))};
1304
1305 // If we are in a cancellable parallel region, barriers are cancellation
1306 // points.
1307 // TODO: Check why we would force simple calls or to ignore the cancel flag.
1308 bool UseCancelBarrier =
1309 !ForceSimpleCall && isLastFinalizationInfoCancellable(DK: OMPD_parallel);
1310
1311 Value *Result = createRuntimeFunctionCall(
1312 Callee: getOrCreateRuntimeFunctionPtr(FnID: UseCancelBarrier
1313 ? OMPRTL___kmpc_cancel_barrier
1314 : OMPRTL___kmpc_barrier),
1315 Args);
1316
1317 if (UseCancelBarrier && CheckCancelFlag)
1318 if (Error Err = emitCancelationCheckImpl(CancelFlag: Result, CanceledDirective: OMPD_parallel))
1319 return Err;
1320
1321 return Builder.saveIP();
1322}
1323
1324OpenMPIRBuilder::InsertPointOrErrorTy
1325OpenMPIRBuilder::createCancel(const LocationDescription &Loc,
1326 Value *IfCondition,
1327 omp::Directive CanceledDirective) {
1328 if (!updateToLocation(Loc))
1329 return Loc.IP;
1330
1331 // LLVM utilities like blocks with terminators.
1332 auto *UI = Builder.CreateUnreachable();
1333
1334 Instruction *ThenTI = UI, *ElseTI = nullptr;
1335 if (IfCondition) {
1336 SplitBlockAndInsertIfThenElse(Cond: IfCondition, SplitBefore: UI, ThenTerm: &ThenTI, ElseTerm: &ElseTI);
1337
1338 // Even if the if condition evaluates to false, this should count as a
1339 // cancellation point
1340 Builder.SetInsertPoint(ElseTI);
1341 auto ElseIP = Builder.saveIP();
1342
1343 InsertPointOrErrorTy IPOrErr = createCancellationPoint(
1344 Loc: LocationDescription{ElseIP, Loc.DL}, CanceledDirective);
1345 if (!IPOrErr)
1346 return IPOrErr;
1347 }
1348
1349 Builder.SetInsertPoint(ThenTI);
1350
1351 Value *CancelKind = nullptr;
1352 switch (CanceledDirective) {
1353#define OMP_CANCEL_KIND(Enum, Str, DirectiveEnum, Value) \
1354 case DirectiveEnum: \
1355 CancelKind = Builder.getInt32(Value); \
1356 break;
1357#include "llvm/Frontend/OpenMP/OMPKinds.def"
1358 default:
1359 llvm_unreachable("Unknown cancel kind!");
1360 }
1361
1362 uint32_t SrcLocStrSize;
1363 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1364 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
1365 Value *Args[] = {Ident, getOrCreateThreadID(Ident), CancelKind};
1366 Value *Result = createRuntimeFunctionCall(
1367 Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_cancel), Args);
1368
1369 // The actual cancel logic is shared with others, e.g., cancel_barriers.
1370 if (Error Err = emitCancelationCheckImpl(CancelFlag: Result, CanceledDirective))
1371 return Err;
1372
1373 // Update the insertion point and remove the terminator we introduced.
1374 Builder.SetInsertPoint(UI->getParent());
1375 UI->eraseFromParent();
1376
1377 return Builder.saveIP();
1378}
1379
1380OpenMPIRBuilder::InsertPointOrErrorTy
1381OpenMPIRBuilder::createCancellationPoint(const LocationDescription &Loc,
1382 omp::Directive CanceledDirective) {
1383 if (!updateToLocation(Loc))
1384 return Loc.IP;
1385
1386 // LLVM utilities like blocks with terminators.
1387 auto *UI = Builder.CreateUnreachable();
1388 Builder.SetInsertPoint(UI);
1389
1390 Value *CancelKind = nullptr;
1391 switch (CanceledDirective) {
1392#define OMP_CANCEL_KIND(Enum, Str, DirectiveEnum, Value) \
1393 case DirectiveEnum: \
1394 CancelKind = Builder.getInt32(Value); \
1395 break;
1396#include "llvm/Frontend/OpenMP/OMPKinds.def"
1397 default:
1398 llvm_unreachable("Unknown cancel kind!");
1399 }
1400
1401 uint32_t SrcLocStrSize;
1402 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1403 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
1404 Value *Args[] = {Ident, getOrCreateThreadID(Ident), CancelKind};
1405 Value *Result = createRuntimeFunctionCall(
1406 Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_cancellationpoint), Args);
1407
1408 // The actual cancel logic is shared with others, e.g., cancel_barriers.
1409 if (Error Err = emitCancelationCheckImpl(CancelFlag: Result, CanceledDirective))
1410 return Err;
1411
1412 // Update the insertion point and remove the terminator we introduced.
1413 Builder.SetInsertPoint(UI->getParent());
1414 UI->eraseFromParent();
1415
1416 return Builder.saveIP();
1417}
1418
1419OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitTargetKernel(
1420 const LocationDescription &Loc, InsertPointTy AllocaIP, Value *&Return,
1421 Value *Ident, Value *DeviceID, Value *NumTeams, Value *NumThreads,
1422 Value *HostPtr, ArrayRef<Value *> KernelArgs) {
1423 if (!updateToLocation(Loc))
1424 return Loc.IP;
1425
1426 Builder.restoreIP(IP: AllocaIP);
1427 auto *KernelArgsPtr =
1428 Builder.CreateAlloca(Ty: OpenMPIRBuilder::KernelArgs, ArraySize: nullptr, Name: "kernel_args");
1429 updateToLocation(Loc);
1430
1431 for (unsigned I = 0, Size = KernelArgs.size(); I != Size; ++I) {
1432 llvm::Value *Arg =
1433 Builder.CreateStructGEP(Ty: OpenMPIRBuilder::KernelArgs, Ptr: KernelArgsPtr, Idx: I);
1434 Builder.CreateAlignedStore(
1435 Val: KernelArgs[I], Ptr: Arg,
1436 Align: M.getDataLayout().getPrefTypeAlign(Ty: KernelArgs[I]->getType()));
1437 }
1438
1439 SmallVector<Value *> OffloadingArgs{Ident, DeviceID, NumTeams,
1440 NumThreads, HostPtr, KernelArgsPtr};
1441
1442 Return = createRuntimeFunctionCall(
1443 Callee: getOrCreateRuntimeFunction(M, FnID: OMPRTL___tgt_target_kernel),
1444 Args: OffloadingArgs);
1445
1446 return Builder.saveIP();
1447}
1448
1449OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitKernelLaunch(
1450 const LocationDescription &Loc, Value *OutlinedFnID,
1451 EmitFallbackCallbackTy EmitTargetCallFallbackCB, TargetKernelArgs &Args,
1452 Value *DeviceID, Value *RTLoc, InsertPointTy AllocaIP) {
1453
1454 if (!updateToLocation(Loc))
1455 return Loc.IP;
1456
1457 // On top of the arrays that were filled up, the target offloading call
1458 // takes as arguments the device id as well as the host pointer. The host
1459 // pointer is used by the runtime library to identify the current target
1460 // region, so it only has to be unique and not necessarily point to
1461 // anything. It could be the pointer to the outlined function that
1462 // implements the target region, but we aren't using that so that the
1463 // compiler doesn't need to keep that, and could therefore inline the host
1464 // function if proven worthwhile during optimization.
1465
1466 // From this point on, we need to have an ID of the target region defined.
1467 assert(OutlinedFnID && "Invalid outlined function ID!");
1468 (void)OutlinedFnID;
1469
1470 // Return value of the runtime offloading call.
1471 Value *Return = nullptr;
1472
1473 // Arguments for the target kernel.
1474 SmallVector<Value *> ArgsVector;
1475 getKernelArgsVector(KernelArgs&: Args, Builder, ArgsVector);
1476
1477 // The target region is an outlined function launched by the runtime
1478 // via calls to __tgt_target_kernel().
1479 //
1480 // Note that on the host and CPU targets, the runtime implementation of
1481 // these calls simply call the outlined function without forking threads.
1482 // The outlined functions themselves have runtime calls to
1483 // __kmpc_fork_teams() and __kmpc_fork() for this purpose, codegen'd by
1484 // the compiler in emitTeamsCall() and emitParallelCall().
1485 //
1486 // In contrast, on the NVPTX target, the implementation of
1487 // __tgt_target_teams() launches a GPU kernel with the requested number
1488 // of teams and threads so no additional calls to the runtime are required.
1489 // Check the error code and execute the host version if required.
1490 Builder.restoreIP(IP: emitTargetKernel(
1491 Loc: Builder, AllocaIP, Return, Ident: RTLoc, DeviceID, NumTeams: Args.NumTeams.front(),
1492 NumThreads: Args.NumThreads.front(), HostPtr: OutlinedFnID, KernelArgs: ArgsVector));
1493
1494 BasicBlock *OffloadFailedBlock =
1495 BasicBlock::Create(Context&: Builder.getContext(), Name: "omp_offload.failed");
1496 BasicBlock *OffloadContBlock =
1497 BasicBlock::Create(Context&: Builder.getContext(), Name: "omp_offload.cont");
1498 Value *Failed = Builder.CreateIsNotNull(Arg: Return);
1499 Builder.CreateCondBr(Cond: Failed, True: OffloadFailedBlock, False: OffloadContBlock);
1500
1501 auto CurFn = Builder.GetInsertBlock()->getParent();
1502 emitBlock(BB: OffloadFailedBlock, CurFn);
1503 InsertPointOrErrorTy AfterIP = EmitTargetCallFallbackCB(Builder.saveIP());
1504 if (!AfterIP)
1505 return AfterIP.takeError();
1506 Builder.restoreIP(IP: *AfterIP);
1507 emitBranch(Target: OffloadContBlock);
1508 emitBlock(BB: OffloadContBlock, CurFn, /*IsFinished=*/true);
1509 return Builder.saveIP();
1510}
1511
1512Error OpenMPIRBuilder::emitCancelationCheckImpl(
1513 Value *CancelFlag, omp::Directive CanceledDirective) {
1514 assert(isLastFinalizationInfoCancellable(CanceledDirective) &&
1515 "Unexpected cancellation!");
1516
1517 // For a cancel barrier we create two new blocks.
1518 BasicBlock *BB = Builder.GetInsertBlock();
1519 BasicBlock *NonCancellationBlock;
1520 if (Builder.GetInsertPoint() == BB->end()) {
1521 // TODO: This branch will not be needed once we moved to the
1522 // OpenMPIRBuilder codegen completely.
1523 NonCancellationBlock = BasicBlock::Create(
1524 Context&: BB->getContext(), Name: BB->getName() + ".cont", Parent: BB->getParent());
1525 } else {
1526 NonCancellationBlock = SplitBlock(Old: BB, SplitPt: &*Builder.GetInsertPoint());
1527 BB->getTerminator()->eraseFromParent();
1528 Builder.SetInsertPoint(BB);
1529 }
1530 BasicBlock *CancellationBlock = BasicBlock::Create(
1531 Context&: BB->getContext(), Name: BB->getName() + ".cncl", Parent: BB->getParent());
1532
1533 // Jump to them based on the return value.
1534 Value *Cmp = Builder.CreateIsNull(Arg: CancelFlag);
1535 Builder.CreateCondBr(Cond: Cmp, True: NonCancellationBlock, False: CancellationBlock,
1536 /* TODO weight */ BranchWeights: nullptr, Unpredictable: nullptr);
1537
1538 // From the cancellation block we finalize all variables and go to the
1539 // post finalization block that is known to the FiniCB callback.
1540 auto &FI = FinalizationStack.back();
1541 Expected<BasicBlock *> FiniBBOrErr = FI.getFiniBB(Builder);
1542 if (!FiniBBOrErr)
1543 return FiniBBOrErr.takeError();
1544 Builder.SetInsertPoint(CancellationBlock);
1545 Builder.CreateBr(Dest: *FiniBBOrErr);
1546
1547 // The continuation block is where code generation continues.
1548 Builder.SetInsertPoint(TheBB: NonCancellationBlock, IP: NonCancellationBlock->begin());
1549 return Error::success();
1550}
1551
1552/// Create wrapper function used to gather the outlined function's argument
1553/// structure from a shared buffer and to forward them to it when running in
1554/// Generic mode.
1555///
1556/// The outlined function is expected to receive 2 integer arguments followed by
1557/// an optional pointer argument to an argument structure holding the rest.
1558static Function *createTargetParallelWrapper(OpenMPIRBuilder *OMPIRBuilder,
1559 Function &OutlinedFn) {
1560 size_t NumArgs = OutlinedFn.arg_size();
1561 assert((NumArgs == 2 || NumArgs == 3) &&
1562 "expected a 2-3 argument parallel outlined function");
1563 bool UseArgStruct = NumArgs == 3;
1564
1565 IRBuilder<> &Builder = OMPIRBuilder->Builder;
1566 IRBuilder<>::InsertPointGuard IPG(Builder);
1567 auto *FnTy = FunctionType::get(Result: Builder.getVoidTy(),
1568 Params: {Builder.getInt16Ty(), Builder.getInt32Ty()},
1569 /*isVarArg=*/false);
1570 auto *WrapperFn =
1571 Function::Create(Ty: FnTy, Linkage: GlobalValue::InternalLinkage,
1572 N: OutlinedFn.getName() + ".wrapper", M&: OMPIRBuilder->M);
1573
1574 WrapperFn->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
1575 WrapperFn->addParamAttr(ArgNo: 0, Kind: Attribute::ZExt);
1576 WrapperFn->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
1577
1578 BasicBlock *EntryBB =
1579 BasicBlock::Create(Context&: OMPIRBuilder->M.getContext(), Name: "entry", Parent: WrapperFn);
1580 Builder.SetInsertPoint(EntryBB);
1581
1582 // Allocation.
1583 Value *AddrAlloca = Builder.CreateAlloca(Ty: Builder.getInt32Ty(),
1584 /*ArraySize=*/nullptr, Name: "addr");
1585 AddrAlloca = Builder.CreatePointerBitCastOrAddrSpaceCast(
1586 V: AddrAlloca, DestTy: Builder.getPtrTy(/*AddrSpace=*/0),
1587 Name: AddrAlloca->getName() + ".ascast");
1588
1589 Value *ZeroAlloca = Builder.CreateAlloca(Ty: Builder.getInt32Ty(),
1590 /*ArraySize=*/nullptr, Name: "zero");
1591 ZeroAlloca = Builder.CreatePointerBitCastOrAddrSpaceCast(
1592 V: ZeroAlloca, DestTy: Builder.getPtrTy(/*AddrSpace=*/0),
1593 Name: ZeroAlloca->getName() + ".ascast");
1594
1595 Value *ArgsAlloca = nullptr;
1596 if (UseArgStruct) {
1597 ArgsAlloca = Builder.CreateAlloca(Ty: Builder.getPtrTy(),
1598 /*ArraySize=*/nullptr, Name: "global_args");
1599 ArgsAlloca = Builder.CreatePointerBitCastOrAddrSpaceCast(
1600 V: ArgsAlloca, DestTy: Builder.getPtrTy(/*AddrSpace=*/0),
1601 Name: ArgsAlloca->getName() + ".ascast");
1602 }
1603
1604 // Initialization.
1605 Builder.CreateStore(Val: WrapperFn->getArg(i: 1), Ptr: AddrAlloca);
1606 Builder.CreateStore(Val: Builder.getInt32(C: 0), Ptr: ZeroAlloca);
1607 if (UseArgStruct) {
1608 Builder.CreateCall(
1609 Callee: OMPIRBuilder->getOrCreateRuntimeFunctionPtr(
1610 FnID: llvm::omp::RuntimeFunction::OMPRTL___kmpc_get_shared_variables),
1611 Args: {ArgsAlloca});
1612 }
1613
1614 SmallVector<Value *, 3> Args{AddrAlloca, ZeroAlloca};
1615
1616 // Load structArg from global_args.
1617 if (UseArgStruct) {
1618 Value *StructArg = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ArgsAlloca);
1619 StructArg = Builder.CreateInBoundsGEP(Ty: Builder.getPtrTy(), Ptr: StructArg,
1620 IdxList: {Builder.getInt64(C: 0)});
1621 StructArg = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: StructArg, Name: "structArg");
1622 Args.push_back(Elt: StructArg);
1623 }
1624
1625 // Call the outlined function holding the parallel body.
1626 Builder.CreateCall(Callee: &OutlinedFn, Args);
1627 Builder.CreateRetVoid();
1628
1629 return WrapperFn;
1630}
1631
1632// Callback used to create OpenMP runtime calls to support
1633// omp parallel clause for the device.
1634// We need to use this callback to replace call to the OutlinedFn in OuterFn
1635// by the call to the OpenMP DeviceRTL runtime function (kmpc_parallel_60)
1636static void targetParallelCallback(
1637 OpenMPIRBuilder *OMPIRBuilder, Function &OutlinedFn, Function *OuterFn,
1638 BasicBlock *OuterAllocaBB, Value *Ident, Value *IfCondition,
1639 Value *NumThreads, Instruction *PrivTID, AllocaInst *PrivTIDAddr,
1640 Value *ThreadID, const SmallVector<Instruction *, 4> &ToBeDeleted) {
1641 assert(OutlinedFn.arg_size() >= 2 &&
1642 "Expected at least tid and bounded tid as arguments");
1643 unsigned NumCapturedVars = OutlinedFn.arg_size() - /* tid & bounded tid */ 2;
1644
1645 // Add some known attributes.
1646 IRBuilder<> &Builder = OMPIRBuilder->Builder;
1647 OutlinedFn.addParamAttr(ArgNo: 0, Kind: Attribute::NoAlias);
1648 OutlinedFn.addParamAttr(ArgNo: 1, Kind: Attribute::NoAlias);
1649 OutlinedFn.addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
1650 OutlinedFn.addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
1651 OutlinedFn.addFnAttr(Kind: Attribute::NoUnwind);
1652
1653 CallInst *CI = cast<CallInst>(Val: OutlinedFn.user_back());
1654 assert(CI && "Expected call instruction to outlined function");
1655 CI->getParent()->setName("omp_parallel");
1656
1657 Builder.SetInsertPoint(CI);
1658 Type *PtrTy = OMPIRBuilder->VoidPtr;
1659
1660 // Add alloca for kernel args
1661 OpenMPIRBuilder ::InsertPointTy CurrentIP = Builder.saveIP();
1662 Builder.SetInsertPoint(TheBB: OuterAllocaBB, IP: OuterAllocaBB->getFirstInsertionPt());
1663 AllocaInst *ArgsAlloca =
1664 Builder.CreateAlloca(Ty: ArrayType::get(ElementType: PtrTy, NumElements: NumCapturedVars));
1665 Value *Args = ArgsAlloca;
1666 // Add address space cast if array for storing arguments is not allocated
1667 // in address space 0
1668 if (ArgsAlloca->getAddressSpace())
1669 Args = Builder.CreatePointerCast(V: ArgsAlloca, DestTy: PtrTy);
1670 Builder.restoreIP(IP: CurrentIP);
1671
1672 // Store captured vars which are used by kmpc_parallel_60
1673 for (unsigned Idx = 0; Idx < NumCapturedVars; Idx++) {
1674 Value *V = *(CI->arg_begin() + 2 + Idx);
1675 Value *StoreAddress = Builder.CreateConstInBoundsGEP2_64(
1676 Ty: ArrayType::get(ElementType: PtrTy, NumElements: NumCapturedVars), Ptr: Args, Idx0: 0, Idx1: Idx);
1677 Builder.CreateStore(Val: V, Ptr: StoreAddress);
1678 }
1679
1680 Value *Cond =
1681 IfCondition ? Builder.CreateSExtOrTrunc(V: IfCondition, DestTy: OMPIRBuilder->Int32)
1682 : Builder.getInt32(C: 1);
1683 Value *NumThreadsArg =
1684 NumThreads ? Builder.CreateZExtOrTrunc(V: NumThreads, DestTy: OMPIRBuilder->Int32)
1685 : Builder.getInt32(C: -1);
1686
1687 // If this is not a Generic kernel, we can skip generating the wrapper.
1688 Value *WrapperFn;
1689 if (isGenericKernel(Fn&: *OuterFn))
1690 WrapperFn = createTargetParallelWrapper(OMPIRBuilder, OutlinedFn);
1691 else
1692 WrapperFn = Constant::getNullValue(Ty: PtrTy);
1693
1694 // Build kmpc_parallel_60 call
1695 Value *Parallel60CallArgs[] = {
1696 /* identifier*/ Ident,
1697 /* global thread num*/ ThreadID,
1698 /* if expression */ Cond,
1699 /* number of threads */ NumThreadsArg,
1700 /* Proc bind */ Builder.getInt32(C: -1),
1701 /* outlined function */ &OutlinedFn,
1702 /* wrapper function */ WrapperFn,
1703 /* arguments of the outlined funciton*/ Args,
1704 /* number of arguments */ Builder.getInt64(C: NumCapturedVars),
1705 /* strict for number of threads */ Builder.getInt32(C: 0)};
1706
1707 FunctionCallee RTLFn =
1708 OMPIRBuilder->getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_parallel_60);
1709
1710 OMPIRBuilder->createRuntimeFunctionCall(Callee: RTLFn, Args: Parallel60CallArgs);
1711
1712 LLVM_DEBUG(dbgs() << "With kmpc_parallel_60 placed: "
1713 << *Builder.GetInsertBlock()->getParent() << "\n");
1714
1715 // Initialize the local TID stack location with the argument value.
1716 Builder.SetInsertPoint(PrivTID);
1717 Function::arg_iterator OutlinedAI = OutlinedFn.arg_begin();
1718 Builder.CreateStore(Val: Builder.CreateLoad(Ty: OMPIRBuilder->Int32, Ptr: OutlinedAI),
1719 Ptr: PrivTIDAddr);
1720
1721 // Remove redundant call to the outlined function.
1722 CI->eraseFromParent();
1723
1724 for (Instruction *I : ToBeDeleted) {
1725 I->eraseFromParent();
1726 }
1727}
1728
1729// Callback used to create OpenMP runtime calls to support
1730// omp parallel clause for the host.
1731// We need to use this callback to replace call to the OutlinedFn in OuterFn
1732// by the call to the OpenMP host runtime function ( __kmpc_fork_call[_if])
1733static void
1734hostParallelCallback(OpenMPIRBuilder *OMPIRBuilder, Function &OutlinedFn,
1735 Function *OuterFn, Value *Ident, Value *IfCondition,
1736 Instruction *PrivTID, AllocaInst *PrivTIDAddr,
1737 const SmallVector<Instruction *, 4> &ToBeDeleted) {
1738 IRBuilder<> &Builder = OMPIRBuilder->Builder;
1739 FunctionCallee RTLFn;
1740 if (IfCondition) {
1741 RTLFn =
1742 OMPIRBuilder->getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_fork_call_if);
1743 } else {
1744 RTLFn =
1745 OMPIRBuilder->getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_fork_call);
1746 }
1747 if (auto *F = dyn_cast<Function>(Val: RTLFn.getCallee())) {
1748 if (!F->hasMetadata(KindID: LLVMContext::MD_callback)) {
1749 LLVMContext &Ctx = F->getContext();
1750 MDBuilder MDB(Ctx);
1751 // Annotate the callback behavior of the __kmpc_fork_call:
1752 // - The callback callee is argument number 2 (microtask).
1753 // - The first two arguments of the callback callee are unknown (-1).
1754 // - All variadic arguments to the __kmpc_fork_call are passed to the
1755 // callback callee.
1756 F->addMetadata(KindID: LLVMContext::MD_callback,
1757 MD&: *MDNode::get(Context&: Ctx, MDs: {MDB.createCallbackEncoding(
1758 CalleeArgNo: 2, Arguments: {-1, -1},
1759 /* VarArgsArePassed */ true)}));
1760 }
1761 }
1762 // Add some known attributes.
1763 OutlinedFn.addParamAttr(ArgNo: 0, Kind: Attribute::NoAlias);
1764 OutlinedFn.addParamAttr(ArgNo: 1, Kind: Attribute::NoAlias);
1765 OutlinedFn.addFnAttr(Kind: Attribute::NoUnwind);
1766
1767 assert(OutlinedFn.arg_size() >= 2 &&
1768 "Expected at least tid and bounded tid as arguments");
1769 unsigned NumCapturedVars = OutlinedFn.arg_size() - /* tid & bounded tid */ 2;
1770
1771 CallInst *CI = cast<CallInst>(Val: OutlinedFn.user_back());
1772 CI->getParent()->setName("omp_parallel");
1773 Builder.SetInsertPoint(CI);
1774
1775 // Build call __kmpc_fork_call[_if](Ident, n, microtask, var1, .., varn);
1776 Value *ForkCallArgs[] = {Ident, Builder.getInt32(C: NumCapturedVars),
1777 &OutlinedFn};
1778
1779 SmallVector<Value *, 16> RealArgs;
1780 RealArgs.append(in_start: std::begin(arr&: ForkCallArgs), in_end: std::end(arr&: ForkCallArgs));
1781 if (IfCondition) {
1782 Value *Cond = Builder.CreateSExtOrTrunc(V: IfCondition, DestTy: OMPIRBuilder->Int32);
1783 RealArgs.push_back(Elt: Cond);
1784 }
1785 RealArgs.append(in_start: CI->arg_begin() + /* tid & bound tid */ 2, in_end: CI->arg_end());
1786
1787 // __kmpc_fork_call_if always expects a void ptr as the last argument
1788 // If there are no arguments, pass a null pointer.
1789 auto PtrTy = OMPIRBuilder->VoidPtr;
1790 if (IfCondition && NumCapturedVars == 0) {
1791 Value *NullPtrValue = Constant::getNullValue(Ty: PtrTy);
1792 RealArgs.push_back(Elt: NullPtrValue);
1793 }
1794
1795 OMPIRBuilder->createRuntimeFunctionCall(Callee: RTLFn, Args: RealArgs);
1796
1797 LLVM_DEBUG(dbgs() << "With fork_call placed: "
1798 << *Builder.GetInsertBlock()->getParent() << "\n");
1799
1800 // Initialize the local TID stack location with the argument value.
1801 Builder.SetInsertPoint(PrivTID);
1802 Function::arg_iterator OutlinedAI = OutlinedFn.arg_begin();
1803 Builder.CreateStore(Val: Builder.CreateLoad(Ty: OMPIRBuilder->Int32, Ptr: OutlinedAI),
1804 Ptr: PrivTIDAddr);
1805
1806 // Remove redundant call to the outlined function.
1807 CI->eraseFromParent();
1808
1809 for (Instruction *I : ToBeDeleted) {
1810 I->eraseFromParent();
1811 }
1812}
1813
1814OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createParallel(
1815 const LocationDescription &Loc, InsertPointTy OuterAllocIP,
1816 ArrayRef<BasicBlock *> OuterDeallocBlocks, BodyGenCallbackTy BodyGenCB,
1817 PrivatizeCallbackTy PrivCB, FinalizeCallbackTy FiniCB, Value *IfCondition,
1818 Value *NumThreads, omp::ProcBindKind ProcBind, bool IsCancellable) {
1819 assert(!isConflictIP(Loc.IP, OuterAllocIP) && "IPs must not be ambiguous");
1820
1821 if (!updateToLocation(Loc))
1822 return Loc.IP;
1823
1824 uint32_t SrcLocStrSize;
1825 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1826 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
1827 const bool NeedThreadID = NumThreads || Config.isTargetDevice() ||
1828 (ProcBind != OMP_PROC_BIND_default);
1829 Value *ThreadID = NeedThreadID ? getOrCreateThreadID(Ident) : nullptr;
1830 // If we generate code for the target device, we need to allocate
1831 // struct for aggregate params in the device default alloca address space.
1832 // OpenMP runtime requires that the params of the extracted functions are
1833 // passed as zero address space pointers. This flag ensures that extracted
1834 // function arguments are declared in zero address space
1835 bool ArgsInZeroAddressSpace = Config.isTargetDevice();
1836
1837 // Build call __kmpc_push_num_threads(&Ident, global_tid, num_threads)
1838 // only if we compile for host side.
1839 if (NumThreads && !Config.isTargetDevice()) {
1840 Value *Args[] = {
1841 Ident, ThreadID,
1842 Builder.CreateIntCast(V: NumThreads, DestTy: Int32, /*isSigned*/ false)};
1843 createRuntimeFunctionCall(
1844 Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_push_num_threads), Args);
1845 }
1846
1847 if (ProcBind != OMP_PROC_BIND_default) {
1848 // Build call __kmpc_push_proc_bind(&Ident, global_tid, proc_bind)
1849 Value *Args[] = {
1850 Ident, ThreadID,
1851 ConstantInt::get(Ty: Int32, V: unsigned(ProcBind), /*isSigned=*/IsSigned: true)};
1852 createRuntimeFunctionCall(
1853 Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_push_proc_bind), Args);
1854 }
1855
1856 BasicBlock *InsertBB = Builder.GetInsertBlock();
1857 Function *OuterFn = InsertBB->getParent();
1858
1859 // Save the outer alloca block because the insertion iterator may get
1860 // invalidated and we still need this later.
1861 BasicBlock *OuterAllocaBlock = OuterAllocIP.getBlock();
1862
1863 // Vector to remember instructions we used only during the modeling but which
1864 // we want to delete at the end.
1865 SmallVector<Instruction *, 4> ToBeDeleted;
1866
1867 // Change the location to the outer alloca insertion point to create and
1868 // initialize the allocas we pass into the parallel region.
1869 InsertPointTy NewOuter(OuterAllocaBlock, OuterAllocaBlock->begin());
1870 Builder.restoreIP(IP: NewOuter);
1871 AllocaInst *TIDAddrAlloca = Builder.CreateAlloca(Ty: Int32, ArraySize: nullptr, Name: "tid.addr");
1872 AllocaInst *ZeroAddrAlloca =
1873 Builder.CreateAlloca(Ty: Int32, ArraySize: nullptr, Name: "zero.addr");
1874 Instruction *TIDAddr = TIDAddrAlloca;
1875 Instruction *ZeroAddr = ZeroAddrAlloca;
1876 if (ArgsInZeroAddressSpace && M.getDataLayout().getAllocaAddrSpace() != 0) {
1877 // Add additional casts to enforce pointers in zero address space
1878 TIDAddr = new AddrSpaceCastInst(
1879 TIDAddrAlloca, PointerType ::get(C&: M.getContext(), AddressSpace: 0), "tid.addr.ascast");
1880 TIDAddr->insertAfter(InsertPos: TIDAddrAlloca->getIterator());
1881 ToBeDeleted.push_back(Elt: TIDAddr);
1882 ZeroAddr = new AddrSpaceCastInst(ZeroAddrAlloca,
1883 PointerType ::get(C&: M.getContext(), AddressSpace: 0),
1884 "zero.addr.ascast");
1885 ZeroAddr->insertAfter(InsertPos: ZeroAddrAlloca->getIterator());
1886 ToBeDeleted.push_back(Elt: ZeroAddr);
1887 }
1888
1889 // We only need TIDAddr and ZeroAddr for modeling purposes to get the
1890 // associated arguments in the outlined function, so we delete them later.
1891 ToBeDeleted.push_back(Elt: TIDAddrAlloca);
1892 ToBeDeleted.push_back(Elt: ZeroAddrAlloca);
1893
1894 // Create an artificial insertion point that will also ensure the blocks we
1895 // are about to split are not degenerated.
1896 auto *UI = new UnreachableInst(Builder.getContext(), InsertBB);
1897
1898 BasicBlock *EntryBB = UI->getParent();
1899 BasicBlock *PRegEntryBB = EntryBB->splitBasicBlock(I: UI, BBName: "omp.par.entry");
1900 BasicBlock *PRegBodyBB = PRegEntryBB->splitBasicBlock(I: UI, BBName: "omp.par.region");
1901 BasicBlock *PRegPreFiniBB =
1902 PRegBodyBB->splitBasicBlock(I: UI, BBName: "omp.par.pre_finalize");
1903 BasicBlock *PRegExitBB = PRegPreFiniBB->splitBasicBlock(I: UI, BBName: "omp.par.exit");
1904
1905 auto FiniCBWrapper = [&](InsertPointTy IP) {
1906 // Hide "open-ended" blocks from the given FiniCB by setting the right jump
1907 // target to the region exit block.
1908 if (IP.getBlock()->end() == IP.getPoint()) {
1909 IRBuilder<>::InsertPointGuard IPG(Builder);
1910 Builder.restoreIP(IP);
1911 Instruction *I = Builder.CreateBr(Dest: PRegExitBB);
1912 IP = InsertPointTy(I->getParent(), I->getIterator());
1913 }
1914 assert(IP.getBlock()->getTerminator()->getNumSuccessors() == 1 &&
1915 IP.getBlock()->getTerminator()->getSuccessor(0) == PRegExitBB &&
1916 "Unexpected insertion point for finalization call!");
1917 return FiniCB(IP);
1918 };
1919
1920 FinalizationStack.push_back(Elt: {FiniCBWrapper, OMPD_parallel, IsCancellable});
1921
1922 // Generate the privatization allocas in the block that will become the entry
1923 // of the outlined function.
1924 Builder.SetInsertPoint(PRegEntryBB->getTerminator());
1925 InsertPointTy InnerAllocaIP = Builder.saveIP();
1926
1927 AllocaInst *PrivTIDAddr =
1928 Builder.CreateAlloca(Ty: Int32, ArraySize: nullptr, Name: "tid.addr.local");
1929 Instruction *PrivTID = Builder.CreateLoad(Ty: Int32, Ptr: PrivTIDAddr, Name: "tid");
1930
1931 // Add some fake uses for OpenMP provided arguments.
1932 ToBeDeleted.push_back(Elt: Builder.CreateLoad(Ty: Int32, Ptr: TIDAddr, Name: "tid.addr.use"));
1933 Instruction *ZeroAddrUse =
1934 Builder.CreateLoad(Ty: Int32, Ptr: ZeroAddr, Name: "zero.addr.use");
1935 ToBeDeleted.push_back(Elt: ZeroAddrUse);
1936
1937 // EntryBB
1938 // |
1939 // V
1940 // PRegionEntryBB <- Privatization allocas are placed here.
1941 // |
1942 // V
1943 // PRegionBodyBB <- BodeGen is invoked here.
1944 // |
1945 // V
1946 // PRegPreFiniBB <- The block we will start finalization from.
1947 // |
1948 // V
1949 // PRegionExitBB <- A common exit to simplify block collection.
1950 //
1951
1952 LLVM_DEBUG(dbgs() << "Before body codegen: " << *OuterFn << "\n");
1953
1954 // Let the caller create the body.
1955 assert(BodyGenCB && "Expected body generation callback!");
1956 InsertPointTy CodeGenIP(PRegBodyBB, PRegBodyBB->begin());
1957 if (Error Err = BodyGenCB(InnerAllocaIP, CodeGenIP, PRegExitBB))
1958 return Err;
1959
1960 LLVM_DEBUG(dbgs() << "After body codegen: " << *OuterFn << "\n");
1961
1962 // If OuterFn is a Generic kernel, we need to use device shared memory to
1963 // allocate argument structures. Otherwise, we use stack allocations as usual.
1964 bool UsesDeviceSharedMemory =
1965 Config.isTargetDevice() && isGenericKernel(Fn&: *OuterFn);
1966 std::unique_ptr<OutlineInfo> OI =
1967 UsesDeviceSharedMemory
1968 ? std::make_unique<DeviceSharedMemOutlineInfo>(args&: *this)
1969 : std::make_unique<OutlineInfo>();
1970
1971 if (Config.isTargetDevice()) {
1972 // Generate OpenMP target specific runtime call
1973 OI->PostOutlineCB = [=, ToBeDeletedVec =
1974 std::move(ToBeDeleted)](Function &OutlinedFn) {
1975 targetParallelCallback(OMPIRBuilder: this, OutlinedFn, OuterFn, OuterAllocaBB: OuterAllocaBlock, Ident,
1976 IfCondition, NumThreads, PrivTID, PrivTIDAddr,
1977 ThreadID, ToBeDeleted: ToBeDeletedVec);
1978 };
1979 } else {
1980 // Generate OpenMP host runtime call
1981 OI->PostOutlineCB = [=, ToBeDeletedVec =
1982 std::move(ToBeDeleted)](Function &OutlinedFn) {
1983 hostParallelCallback(OMPIRBuilder: this, OutlinedFn, OuterFn, Ident, IfCondition,
1984 PrivTID, PrivTIDAddr, ToBeDeleted: ToBeDeletedVec);
1985 };
1986 }
1987
1988 OI->FixUpNonEntryAllocas = true;
1989 OI->OuterAllocBB = OuterAllocaBlock;
1990 OI->EntryBB = PRegEntryBB;
1991 OI->ExitBB = PRegExitBB;
1992 OI->OuterDeallocBBs.reserve(N: OuterDeallocBlocks.size());
1993 copy(Range&: OuterDeallocBlocks, Out: OI->OuterDeallocBBs.end());
1994
1995 SmallPtrSet<BasicBlock *, 32> ParallelRegionBlockSet;
1996 SmallVector<BasicBlock *, 32> Blocks;
1997 OI->collectBlocks(BlockSet&: ParallelRegionBlockSet, BlockVector&: Blocks);
1998
1999 CodeExtractorAnalysisCache CEAC(*OuterFn);
2000 CodeExtractor Extractor(Blocks, /* DominatorTree */ nullptr,
2001 /* AggregateArgs */ false,
2002 /* BlockFrequencyInfo */ nullptr,
2003 /* BranchProbabilityInfo */ nullptr,
2004 /* AssumptionCache */ nullptr,
2005 /* AllowVarArgs */ true,
2006 /* AllowAlloca */ true,
2007 /* AllocationBlock */ OuterAllocaBlock,
2008 /* DeallocationBlocks */ {},
2009 /* Suffix */ ".omp_par", ArgsInZeroAddressSpace);
2010
2011 // Find inputs to, outputs from the code region.
2012 BasicBlock *CommonExit = nullptr;
2013 SetVector<Value *> Inputs, Outputs, SinkingCands, HoistingCands;
2014 Extractor.findAllocas(CEAC, SinkCands&: SinkingCands, HoistCands&: HoistingCands, ExitBlock&: CommonExit);
2015
2016 Extractor.findInputsOutputs(Inputs, Outputs, Allocas: SinkingCands,
2017 /*CollectGlobalInputs=*/true);
2018
2019 Inputs.remove_if(P: [&](Value *I) {
2020 if (auto *GV = dyn_cast_if_present<GlobalVariable>(Val: I))
2021 return GV->getValueType() == OpenMPIRBuilder::Ident;
2022
2023 return false;
2024 });
2025
2026 LLVM_DEBUG(dbgs() << "Before privatization: " << *OuterFn << "\n");
2027
2028 FunctionCallee TIDRTLFn =
2029 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_global_thread_num);
2030
2031 auto PrivHelper = [&](Value &V) -> Error {
2032 if (&V == TIDAddr || &V == ZeroAddr) {
2033 OI->ExcludeArgsFromAggregate.push_back(Elt: &V);
2034 return Error::success();
2035 }
2036
2037 SetVector<Use *> Uses;
2038 for (Use &U : V.uses())
2039 if (auto *UserI = dyn_cast<Instruction>(Val: U.getUser()))
2040 if (ParallelRegionBlockSet.count(Ptr: UserI->getParent()))
2041 Uses.insert(X: &U);
2042
2043 // __kmpc_fork_call expects extra arguments as pointers. If the input
2044 // already has a pointer type, everything is fine. Otherwise, store the
2045 // value onto stack and load it back inside the to-be-outlined region. This
2046 // will ensure only the pointer will be passed to the function.
2047 // FIXME: if there are more than 15 trailing arguments, they must be
2048 // additionally packed in a struct.
2049 Value *Inner = &V;
2050 if (!V.getType()->isPointerTy()) {
2051 IRBuilder<>::InsertPointGuard Guard(Builder);
2052 LLVM_DEBUG(llvm::dbgs() << "Forwarding input as pointer: " << V << "\n");
2053
2054 Builder.restoreIP(IP: OuterAllocIP);
2055 Value *Ptr;
2056 if (UsesDeviceSharedMemory) {
2057 // Use device shared memory instead, if needed.
2058 Ptr = createOMPAllocShared(Loc: OuterAllocIP, VarType: V.getType(),
2059 Name: V.getName() + ".reloaded");
2060 for (BasicBlock *DeallocBlock : OuterDeallocBlocks)
2061 createOMPFreeShared(
2062 Loc: InsertPointTy(DeallocBlock, DeallocBlock->getFirstInsertionPt()),
2063 Addr: Ptr, VarType: V.getType());
2064 } else {
2065 Ptr = Builder.CreateAlloca(Ty: V.getType(), ArraySize: nullptr,
2066 Name: V.getName() + ".reloaded");
2067 }
2068
2069 // Store to stack at end of the block that currently branches to the entry
2070 // block of the to-be-outlined region.
2071 Builder.SetInsertPoint(TheBB: InsertBB,
2072 IP: InsertBB->getTerminator()->getIterator());
2073 Builder.CreateStore(Val: &V, Ptr);
2074
2075 // Load back next to allocations in the to-be-outlined region.
2076 Builder.restoreIP(IP: InnerAllocaIP);
2077 Inner = Builder.CreateLoad(Ty: V.getType(), Ptr);
2078 }
2079
2080 Value *ReplacementValue = nullptr;
2081 CallInst *CI = dyn_cast<CallInst>(Val: &V);
2082 if (CI && CI->getCalledFunction() == TIDRTLFn.getCallee()) {
2083 ReplacementValue = PrivTID;
2084 } else {
2085 InsertPointOrErrorTy AfterIP =
2086 PrivCB(InnerAllocaIP, Builder.saveIP(), V, *Inner, ReplacementValue);
2087 if (!AfterIP)
2088 return AfterIP.takeError();
2089 Builder.restoreIP(IP: *AfterIP);
2090 InnerAllocaIP = {
2091 InnerAllocaIP.getBlock(),
2092 InnerAllocaIP.getBlock()->getTerminator()->getIterator()};
2093
2094 assert(ReplacementValue &&
2095 "Expected copy/create callback to set replacement value!");
2096 if (ReplacementValue == &V)
2097 return Error::success();
2098 }
2099
2100 for (Use *UPtr : Uses)
2101 UPtr->set(ReplacementValue);
2102
2103 return Error::success();
2104 };
2105
2106 // Reset the inner alloca insertion as it will be used for loading the values
2107 // wrapped into pointers before passing them into the to-be-outlined region.
2108 // Configure it to insert immediately after the fake use of zero address so
2109 // that they are available in the generated body and so that the
2110 // OpenMP-related values (thread ID and zero address pointers) remain leading
2111 // in the argument list.
2112 InnerAllocaIP = IRBuilder<>::InsertPoint(
2113 ZeroAddrUse->getParent(), ZeroAddrUse->getNextNode()->getIterator());
2114
2115 // Reset the outer alloca insertion point to the entry of the relevant block
2116 // in case it was invalidated.
2117 OuterAllocIP = IRBuilder<>::InsertPoint(
2118 OuterAllocaBlock, OuterAllocaBlock->getFirstInsertionPt());
2119
2120 for (Value *Input : Inputs) {
2121 LLVM_DEBUG(dbgs() << "Captured input: " << *Input << "\n");
2122 if (Error Err = PrivHelper(*Input))
2123 return Err;
2124 }
2125 LLVM_DEBUG({
2126 for (Value *Output : Outputs)
2127 LLVM_DEBUG(dbgs() << "Captured output: " << *Output << "\n");
2128 });
2129 assert(Outputs.empty() &&
2130 "OpenMP outlining should not produce live-out values!");
2131
2132 LLVM_DEBUG(dbgs() << "After privatization: " << *OuterFn << "\n");
2133 LLVM_DEBUG({
2134 for (auto *BB : Blocks)
2135 dbgs() << " PBR: " << BB->getName() << "\n";
2136 });
2137
2138 // Adjust the finalization stack, verify the adjustment, and call the
2139 // finalize function a last time to finalize values between the pre-fini
2140 // block and the exit block if we left the parallel "the normal way".
2141 auto FiniInfo = FinalizationStack.pop_back_val();
2142 (void)FiniInfo;
2143 assert(FiniInfo.DK == OMPD_parallel &&
2144 "Unexpected finalization stack state!");
2145
2146 Instruction *PRegPreFiniTI = PRegPreFiniBB->getTerminator();
2147
2148 InsertPointTy PreFiniIP(PRegPreFiniBB, PRegPreFiniTI->getIterator());
2149 Expected<BasicBlock *> FiniBBOrErr = FiniInfo.getFiniBB(Builder);
2150 if (!FiniBBOrErr)
2151 return FiniBBOrErr.takeError();
2152 {
2153 IRBuilderBase::InsertPointGuard Guard(Builder);
2154 Builder.restoreIP(IP: PreFiniIP);
2155 Builder.CreateBr(Dest: *FiniBBOrErr);
2156 // There's currently a branch to omp.par.exit. Delete it. We will get there
2157 // via the fini block
2158 if (Instruction *Term = Builder.GetInsertBlock()->getTerminator())
2159 Term->eraseFromParent();
2160 }
2161
2162 // Register the outlined info.
2163 addOutlineInfo(OI: std::move(OI));
2164
2165 InsertPointTy AfterIP(UI->getParent(), UI->getParent()->end());
2166 UI->eraseFromParent();
2167
2168 return AfterIP;
2169}
2170
2171void OpenMPIRBuilder::emitFlush(const LocationDescription &Loc) {
2172 // Build call void __kmpc_flush(ident_t *loc)
2173 uint32_t SrcLocStrSize;
2174 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2175 Value *Args[] = {getOrCreateIdent(SrcLocStr, SrcLocStrSize)};
2176
2177 createRuntimeFunctionCall(Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_flush),
2178 Args);
2179}
2180
2181void OpenMPIRBuilder::createFlush(const LocationDescription &Loc) {
2182 if (!updateToLocation(Loc))
2183 return;
2184 emitFlush(Loc);
2185}
2186
2187void OpenMPIRBuilder::emitTaskyieldImpl(const LocationDescription &Loc) {
2188 // Build call __kmpc_omp_taskyield(loc, thread_id, 0);
2189 uint32_t SrcLocStrSize;
2190 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2191 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2192 Constant *I32Null = ConstantInt::getNullValue(Ty: Int32);
2193 Value *Args[] = {Ident, getOrCreateThreadID(Ident), I32Null};
2194
2195 createRuntimeFunctionCall(
2196 Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_taskyield), Args);
2197}
2198
2199void OpenMPIRBuilder::createTaskyield(const LocationDescription &Loc) {
2200 if (!updateToLocation(Loc))
2201 return;
2202 emitTaskyieldImpl(Loc);
2203}
2204
2205void OpenMPIRBuilder::emitTaskDependency(IRBuilderBase &Builder, Value *Entry,
2206 const DependData &Dep) {
2207 // Store the pointer to the variable
2208 Value *Addr = Builder.CreateStructGEP(
2209 Ty: DependInfo, Ptr: Entry,
2210 Idx: static_cast<unsigned int>(RTLDependInfoFields::BaseAddr));
2211 Value *DepValPtr = Builder.CreatePtrToInt(V: Dep.DepVal, DestTy: SizeTy);
2212 Builder.CreateStore(Val: DepValPtr, Ptr: Addr);
2213 // Store the size of the variable
2214 Value *Size = Builder.CreateStructGEP(
2215 Ty: DependInfo, Ptr: Entry, Idx: static_cast<unsigned int>(RTLDependInfoFields::Len));
2216 Builder.CreateStore(
2217 Val: ConstantInt::get(Ty: SizeTy,
2218 V: M.getDataLayout().getTypeStoreSize(Ty: Dep.DepValueType)),
2219 Ptr: Size);
2220 // Store the dependency kind
2221 Value *Flags = Builder.CreateStructGEP(
2222 Ty: DependInfo, Ptr: Entry, Idx: static_cast<unsigned int>(RTLDependInfoFields::Flags));
2223 Builder.CreateStore(Val: ConstantInt::get(Ty: Builder.getInt8Ty(),
2224 V: static_cast<unsigned int>(Dep.DepKind)),
2225 Ptr: Flags);
2226}
2227
2228// Processes the dependencies in Dependencies and does the following
2229// - Allocates space on the stack of an array of DependInfo objects
2230// - Populates each DependInfo object with relevant information of
2231// the corresponding dependence.
2232// - All code is inserted in the entry block of the current function.
2233static Value *emitTaskDependencies(
2234 OpenMPIRBuilder &OMPBuilder,
2235 const SmallVectorImpl<OpenMPIRBuilder::DependData> &Dependencies) {
2236 // Early return if we have no dependencies to process
2237 if (Dependencies.empty())
2238 return nullptr;
2239
2240 // Given a vector of DependData objects, in this function we create an
2241 // array on the stack that holds kmp_depend_info objects corresponding
2242 // to each dependency. This is then passed to the OpenMP runtime.
2243 // For example, if there are 'n' dependencies then the following psedo
2244 // code is generated. Assume the first dependence is on a variable 'a'
2245 //
2246 // \code{c}
2247 // DepArray = alloc(n x sizeof(kmp_depend_info);
2248 // idx = 0;
2249 // DepArray[idx].base_addr = ptrtoint(&a);
2250 // DepArray[idx].len = 8;
2251 // DepArray[idx].flags = Dep.DepKind; /*(See OMPContants.h for DepKind)*/
2252 // ++idx;
2253 // DepArray[idx].base_addr = ...;
2254 // \endcode
2255
2256 IRBuilderBase &Builder = OMPBuilder.Builder;
2257 Type *DependInfo = OMPBuilder.DependInfo;
2258
2259 Value *DepArray = nullptr;
2260 OpenMPIRBuilder::InsertPointTy OldIP = Builder.saveIP();
2261 Builder.SetInsertPoint(
2262 OldIP.getBlock()->getParent()->getEntryBlock().getTerminator());
2263
2264 Type *DepArrayTy = ArrayType::get(ElementType: DependInfo, NumElements: Dependencies.size());
2265 DepArray = Builder.CreateAlloca(Ty: DepArrayTy, ArraySize: nullptr, Name: ".dep.arr.addr");
2266
2267 Builder.restoreIP(IP: OldIP);
2268
2269 for (const auto &[DepIdx, Dep] : enumerate(First: Dependencies)) {
2270 Value *Base =
2271 Builder.CreateConstInBoundsGEP2_64(Ty: DepArrayTy, Ptr: DepArray, Idx0: 0, Idx1: DepIdx);
2272 OMPBuilder.emitTaskDependency(Builder, Entry: Base, Dep);
2273 }
2274 return DepArray;
2275}
2276
2277void OpenMPIRBuilder::emitTaskwaitImpl(const LocationDescription &Loc) {
2278 // Build call kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
2279 // global_tid);
2280 uint32_t SrcLocStrSize;
2281 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2282 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2283 Value *Args[] = {Ident, getOrCreateThreadID(Ident)};
2284
2285 // Ignore return result until untied tasks are supported.
2286 createRuntimeFunctionCall(
2287 Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_taskwait), Args);
2288}
2289
2290void OpenMPIRBuilder::createTaskwait(const LocationDescription &Loc,
2291 DependenciesInfo Dependencies) {
2292 if (!updateToLocation(Loc))
2293 return;
2294
2295 Value *DepArray = nullptr;
2296 Type *DepArrayTy = nullptr;
2297 Value *NumDeps = nullptr;
2298 if (Dependencies.DepArray) {
2299 DepArray = Dependencies.DepArray;
2300 NumDeps = Dependencies.NumDeps;
2301 } else if (!Dependencies.Deps.empty()) {
2302 InsertPointTy OldIP = Builder.saveIP();
2303 BasicBlock &entryBB =
2304 Builder.GetInsertBlock()->getParent()->getEntryBlock();
2305 Builder.SetInsertPoint(TheBB: &entryBB, IP: entryBB.getFirstInsertionPt());
2306
2307 DepArrayTy = ArrayType::get(ElementType: DependInfo, NumElements: Dependencies.Deps.size());
2308 DepArray = Builder.CreateAlloca(Ty: DepArrayTy, ArraySize: nullptr, Name: ".dep.arr.addr");
2309 NumDeps = Builder.getInt32(C: Dependencies.Deps.size());
2310
2311 Builder.restoreIP(IP: OldIP);
2312 for (const auto &[DepIdx, Dep] : enumerate(First&: Dependencies.Deps)) {
2313 Value *Base =
2314 Builder.CreateConstInBoundsGEP2_64(Ty: DepArrayTy, Ptr: DepArray, Idx0: 0, Idx1: DepIdx);
2315 this->emitTaskDependency(Builder, Entry: Base, Dep);
2316 }
2317 }
2318
2319 if (DepArray) {
2320 uint32_t SrcLocStrSize;
2321 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2322 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2323 Value *Args[] = {
2324 Ident,
2325 getOrCreateThreadID(Ident),
2326 NumDeps,
2327 DepArray,
2328 ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0),
2329 ConstantPointerNull::get(T: PointerType::getUnqual(C&: M.getContext())),
2330 ConstantInt::get(Ty: Builder.getInt32Ty(), V: false)};
2331 createRuntimeFunctionCall(
2332 Callee: getOrCreateRuntimeFunctionPtr(
2333 FnID: omp::RuntimeFunction::OMPRTL___kmpc_omp_taskwait_deps_51),
2334 Args);
2335 } else {
2336 emitTaskwaitImpl(Loc);
2337 }
2338}
2339
2340/// Create the task duplication function passed to kmpc_taskloop.
2341Expected<Value *> OpenMPIRBuilder::createTaskDuplicationFunction(
2342 Type *PrivatesTy, int32_t PrivatesIndex, TaskDupCallbackTy DupCB) {
2343 unsigned ProgramAddressSpace = M.getDataLayout().getProgramAddressSpace();
2344 if (!DupCB)
2345 return Constant::getNullValue(
2346 Ty: PointerType::get(C&: Builder.getContext(), AddressSpace: ProgramAddressSpace));
2347
2348 // From OpenMP Runtime p_task_dup_t:
2349 // Routine optionally generated by the compiler for setting the lastprivate
2350 // flag and calling needed constructors for private/firstprivate objects (used
2351 // to form taskloop tasks from pattern task) Parameters: dest task, src task,
2352 // lastprivate flag.
2353 // typedef void (*p_task_dup_t)(kmp_task_t *, kmp_task_t *, kmp_int32);
2354
2355 auto *VoidPtrTy = PointerType::get(C&: Builder.getContext(), AddressSpace: ProgramAddressSpace);
2356
2357 FunctionType *DupFuncTy = FunctionType::get(
2358 Result: Builder.getVoidTy(), Params: {VoidPtrTy, VoidPtrTy, Builder.getInt32Ty()},
2359 /*isVarArg=*/false);
2360
2361 Function *DupFunction = Function::Create(Ty: DupFuncTy, Linkage: Function::InternalLinkage,
2362 N: "omp_taskloop_dup", M);
2363 Value *DestTaskArg = DupFunction->getArg(i: 0);
2364 Value *SrcTaskArg = DupFunction->getArg(i: 1);
2365 Value *LastprivateFlagArg = DupFunction->getArg(i: 2);
2366 DestTaskArg->setName("dest_task");
2367 SrcTaskArg->setName("src_task");
2368 LastprivateFlagArg->setName("lastprivate_flag");
2369
2370 IRBuilderBase::InsertPointGuard Guard(Builder);
2371 Builder.SetInsertPoint(
2372 BasicBlock::Create(Context&: Builder.getContext(), Name: "entry", Parent: DupFunction));
2373
2374 auto GetTaskContextPtrFromArg = [&](Value *Arg) -> Value * {
2375 Type *TaskWithPrivatesTy =
2376 StructType::get(Context&: Builder.getContext(), Elements: {Task, PrivatesTy});
2377 Value *TaskPrivates = Builder.CreateGEP(
2378 Ty: TaskWithPrivatesTy, Ptr: Arg, IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: 1)});
2379 Value *ContextPtr = Builder.CreateGEP(
2380 Ty: PrivatesTy, Ptr: TaskPrivates,
2381 IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: PrivatesIndex)});
2382 return ContextPtr;
2383 };
2384
2385 Value *DestTaskContextPtr = GetTaskContextPtrFromArg(DestTaskArg);
2386 Value *SrcTaskContextPtr = GetTaskContextPtrFromArg(SrcTaskArg);
2387
2388 DestTaskContextPtr->setName("destPtr");
2389 SrcTaskContextPtr->setName("srcPtr");
2390
2391 InsertPointTy AllocaIP(&DupFunction->getEntryBlock(),
2392 DupFunction->getEntryBlock().begin());
2393 InsertPointTy CodeGenIP = Builder.saveIP();
2394 Expected<IRBuilderBase::InsertPoint> AfterIPOrError =
2395 DupCB(AllocaIP, CodeGenIP, DestTaskContextPtr, SrcTaskContextPtr);
2396 if (!AfterIPOrError)
2397 return AfterIPOrError.takeError();
2398 Builder.restoreIP(IP: *AfterIPOrError);
2399
2400 Builder.CreateRetVoid();
2401
2402 return DupFunction;
2403}
2404
2405OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTaskloop(
2406 const LocationDescription &Loc, InsertPointTy AllocaIP,
2407 ArrayRef<BasicBlock *> DeallocBlocks, BodyGenCallbackTy BodyGenCB,
2408 llvm::function_ref<llvm::Expected<llvm::CanonicalLoopInfo *>()> LoopInfo,
2409 Value *LBVal, Value *UBVal, Value *StepVal, bool Untied, Value *IfCond,
2410 Value *GrainSize, bool NoGroup, int Sched, Value *Final, bool Mergeable,
2411 Value *Priority, uint64_t NumOfCollapseLoops, TaskDupCallbackTy DupCB,
2412 Value *TaskContextStructPtrVal) {
2413
2414 if (!updateToLocation(Loc))
2415 return InsertPointTy();
2416
2417 uint32_t SrcLocStrSize;
2418 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2419 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2420
2421 BasicBlock *TaskloopExitBB =
2422 splitBB(Builder, /*CreateBranch=*/true, Name: "taskloop.exit");
2423 BasicBlock *TaskloopBodyBB =
2424 splitBB(Builder, /*CreateBranch=*/true, Name: "taskloop.body");
2425 BasicBlock *TaskloopAllocaBB =
2426 splitBB(Builder, /*CreateBranch=*/true, Name: "taskloop.alloca");
2427
2428 InsertPointTy TaskloopAllocaIP =
2429 InsertPointTy(TaskloopAllocaBB, TaskloopAllocaBB->begin());
2430 InsertPointTy TaskloopBodyIP =
2431 InsertPointTy(TaskloopBodyBB, TaskloopBodyBB->begin());
2432
2433 if (Error Err = BodyGenCB(TaskloopAllocaIP, TaskloopBodyIP, TaskloopExitBB))
2434 return Err;
2435
2436 llvm::Expected<llvm::CanonicalLoopInfo *> result = LoopInfo();
2437 if (!result) {
2438 return result.takeError();
2439 }
2440
2441 llvm::CanonicalLoopInfo *CLI = result.get();
2442 auto OI = std::make_unique<OutlineInfo>();
2443 OI->EntryBB = TaskloopAllocaBB;
2444 OI->OuterAllocBB = AllocaIP.getBlock();
2445 OI->ExitBB = TaskloopExitBB;
2446 OI->OuterDeallocBBs.reserve(N: DeallocBlocks.size());
2447 copy(Range&: DeallocBlocks, Out: OI->OuterDeallocBBs.end());
2448
2449 // Add the thread ID argument.
2450 SmallVector<Instruction *> ToBeDeleted;
2451 // dummy instruction to be used as a fake argument
2452 OI->ExcludeArgsFromAggregate.push_back(Elt: createFakeIntVal(
2453 Builder, OuterAllocaIP: AllocaIP, ToBeDeleted, InnerAllocaIP: TaskloopAllocaIP, Name: "global.tid", AsPtr: false));
2454 Value *FakeLB = createFakeIntVal(Builder, OuterAllocaIP: AllocaIP, ToBeDeleted,
2455 InnerAllocaIP: TaskloopAllocaIP, Name: "lb", AsPtr: false, Is64Bit: true);
2456 Value *FakeUB = createFakeIntVal(Builder, OuterAllocaIP: AllocaIP, ToBeDeleted,
2457 InnerAllocaIP: TaskloopAllocaIP, Name: "ub", AsPtr: false, Is64Bit: true);
2458 Value *FakeStep = createFakeIntVal(Builder, OuterAllocaIP: AllocaIP, ToBeDeleted,
2459 InnerAllocaIP: TaskloopAllocaIP, Name: "step", AsPtr: false, Is64Bit: true);
2460 // For Taskloop, we want to force the bounds being the first 3 inputs in the
2461 // aggregate struct
2462 OI->Inputs.insert(X: FakeLB);
2463 OI->Inputs.insert(X: FakeUB);
2464 OI->Inputs.insert(X: FakeStep);
2465 if (TaskContextStructPtrVal)
2466 OI->Inputs.insert(X: TaskContextStructPtrVal);
2467 assert(((TaskContextStructPtrVal && DupCB) ||
2468 (!TaskContextStructPtrVal && !DupCB)) &&
2469 "Task context struct ptr and duplication callback must be both set "
2470 "or both null");
2471
2472 // It isn't safe to run the duplication bodygen callback inside the post
2473 // outlining callback so this has to be run now before we know the real task
2474 // shareds structure type.
2475 unsigned ProgramAddressSpace = M.getDataLayout().getProgramAddressSpace();
2476 Type *PointerTy = PointerType::get(C&: Builder.getContext(), AddressSpace: ProgramAddressSpace);
2477 Type *FakeSharedsTy = StructType::get(
2478 Context&: Builder.getContext(),
2479 Elements: {FakeLB->getType(), FakeUB->getType(), FakeStep->getType(), PointerTy});
2480 Expected<Value *> TaskDupFnOrErr = createTaskDuplicationFunction(
2481 PrivatesTy: FakeSharedsTy,
2482 /*PrivatesIndex: the pointer after the three indices above*/ PrivatesIndex: 3, DupCB);
2483 if (!TaskDupFnOrErr) {
2484 return TaskDupFnOrErr.takeError();
2485 }
2486 Value *TaskDupFn = *TaskDupFnOrErr;
2487
2488 OI->PostOutlineCB = [this, Ident, LBVal, UBVal, StepVal, Untied,
2489 TaskloopAllocaBB, CLI, TaskDupFn, ToBeDeleted, IfCond,
2490 GrainSize, NoGroup, Sched, FakeLB, FakeUB, FakeStep,
2491 FakeSharedsTy, Final, Mergeable, Priority,
2492 NumOfCollapseLoops](Function &OutlinedFn) mutable {
2493 // Replace the Stale CI by appropriate RTL function call.
2494 assert(OutlinedFn.hasOneUse() &&
2495 "there must be a single user for the outlined function");
2496 CallInst *StaleCI = cast<CallInst>(Val: OutlinedFn.user_back());
2497
2498 /* Create the casting for the Bounds Values that can be used when outlining
2499 * to replace the uses of the fakes with real values */
2500 BasicBlock *CodeReplBB = StaleCI->getParent();
2501 Builder.SetInsertPoint(CodeReplBB->getFirstInsertionPt());
2502 Value *CastedLBVal =
2503 Builder.CreateIntCast(V: LBVal, DestTy: Builder.getInt64Ty(), isSigned: true, Name: "lb64");
2504 Value *CastedUBVal =
2505 Builder.CreateIntCast(V: UBVal, DestTy: Builder.getInt64Ty(), isSigned: true, Name: "ub64");
2506 Value *CastedStepVal =
2507 Builder.CreateIntCast(V: StepVal, DestTy: Builder.getInt64Ty(), isSigned: true, Name: "step64");
2508
2509 Builder.SetInsertPoint(StaleCI);
2510
2511 // Gather the arguments for emitting the runtime call for
2512 // @__kmpc_omp_task_alloc
2513 Function *TaskAllocFn =
2514 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_alloc);
2515
2516 Value *ThreadID = getOrCreateThreadID(Ident);
2517
2518 if (!NoGroup) {
2519 // Emit runtime call for @__kmpc_taskgroup
2520 Function *TaskgroupFn =
2521 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_taskgroup);
2522 Builder.CreateCall(Callee: TaskgroupFn, Args: {Ident, ThreadID});
2523 }
2524
2525 // `flags` Argument Configuration
2526 // Task is tied if (Flags & 1) == 1.
2527 // Task is untied if (Flags & 1) == 0.
2528 // Task is final if (Flags & 2) == 2.
2529 // Task is not final if (Flags & 2) == 0.
2530 // Task is mergeable if (Flags & 4) == 4.
2531 // Task is not mergeable if (Flags & 4) == 0.
2532 // Task is priority if (Flags & 32) == 32.
2533 // Task is not priority if (Flags & 32) == 0.
2534 Value *Flags = Builder.getInt32(C: Untied ? 0 : 1);
2535 if (Final)
2536 Flags = Builder.CreateOr(LHS: Builder.getInt32(C: 2), RHS: Flags);
2537 if (Mergeable)
2538 Flags = Builder.CreateOr(LHS: Builder.getInt32(C: 4), RHS: Flags);
2539 if (Priority)
2540 Flags = Builder.CreateOr(LHS: Builder.getInt32(C: 32), RHS: Flags);
2541
2542 Value *TaskSize = Builder.getInt64(
2543 C: divideCeil(Numerator: M.getDataLayout().getTypeSizeInBits(Ty: Task), Denominator: 8));
2544
2545 AllocaInst *ArgStructAlloca =
2546 dyn_cast<AllocaInst>(Val: StaleCI->getArgOperand(i: 1));
2547 assert(ArgStructAlloca &&
2548 "Unable to find the alloca instruction corresponding to arguments "
2549 "for extracted function");
2550 std::optional<TypeSize> ArgAllocSize =
2551 ArgStructAlloca->getAllocationSize(DL: M.getDataLayout());
2552 assert(ArgAllocSize &&
2553 "Unable to determine size of arguments for extracted function");
2554 Value *SharedsSize = Builder.getInt64(C: ArgAllocSize->getFixedValue());
2555
2556 // Emit the @__kmpc_omp_task_alloc runtime call
2557 // The runtime call returns a pointer to an area where the task captured
2558 // variables must be copied before the task is run (TaskData)
2559 CallInst *TaskData = Builder.CreateCall(
2560 Callee: TaskAllocFn, Args: {/*loc_ref=*/Ident, /*gtid=*/ThreadID, /*flags=*/Flags,
2561 /*sizeof_task=*/TaskSize, /*sizeof_shared=*/SharedsSize,
2562 /*task_func=*/&OutlinedFn});
2563
2564 Value *Shareds = StaleCI->getArgOperand(i: 1);
2565 Align Alignment = TaskData->getPointerAlignment(DL: M.getDataLayout());
2566 Value *TaskShareds = Builder.CreateLoad(Ty: VoidPtr, Ptr: TaskData);
2567 Builder.CreateMemCpy(Dst: TaskShareds, DstAlign: Alignment, Src: Shareds, SrcAlign: Alignment,
2568 Size: SharedsSize);
2569 // Get the pointer to loop lb, ub, step from task ptr
2570 // and set up the lowerbound,upperbound and step values
2571 llvm::Value *Lb = Builder.CreateGEP(
2572 Ty: FakeSharedsTy, Ptr: TaskShareds, IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: 0)});
2573
2574 llvm::Value *Ub = Builder.CreateGEP(
2575 Ty: FakeSharedsTy, Ptr: TaskShareds, IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: 1)});
2576
2577 llvm::Value *Step = Builder.CreateGEP(
2578 Ty: FakeSharedsTy, Ptr: TaskShareds, IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: 2)});
2579 llvm::Value *Loadstep = Builder.CreateLoad(Ty: Builder.getInt64Ty(), Ptr: Step);
2580
2581 // set up the arguments for emitting kmpc_taskloop runtime call
2582 // setting values for ifval, nogroup, sched, grainsize, task_dup
2583 Value *IfCondVal =
2584 IfCond ? Builder.CreateIntCast(V: IfCond, DestTy: Builder.getInt32Ty(), isSigned: true)
2585 : Builder.getInt32(C: 1);
2586 // As __kmpc_taskgroup is called manually in OMPIRBuilder, NoGroupVal should
2587 // always be 1 when calling __kmpc_taskloop to ensure it is not called again
2588 Value *NoGroupVal = Builder.getInt32(C: 1);
2589 Value *SchedVal = Builder.getInt32(C: Sched);
2590 Value *GrainSizeVal =
2591 GrainSize ? Builder.CreateIntCast(V: GrainSize, DestTy: Builder.getInt64Ty(), isSigned: true)
2592 : Builder.getInt64(C: 0);
2593 Value *TaskDup = TaskDupFn;
2594
2595 Value *Args[] = {Ident, ThreadID, TaskData, IfCondVal, Lb, Ub,
2596 Loadstep, NoGroupVal, SchedVal, GrainSizeVal, TaskDup};
2597
2598 // taskloop runtime call
2599 Function *TaskloopFn =
2600 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_taskloop);
2601 Builder.CreateCall(Callee: TaskloopFn, Args);
2602
2603 // Emit the @__kmpc_end_taskgroup runtime call to end the taskgroup if
2604 // nogroup is not defined
2605 if (!NoGroup) {
2606 Function *EndTaskgroupFn =
2607 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_end_taskgroup);
2608 Builder.CreateCall(Callee: EndTaskgroupFn, Args: {Ident, ThreadID});
2609 }
2610
2611 StaleCI->eraseFromParent();
2612
2613 Builder.SetInsertPoint(TheBB: TaskloopAllocaBB, IP: TaskloopAllocaBB->begin());
2614
2615 LoadInst *SharedsOutlined =
2616 Builder.CreateLoad(Ty: VoidPtr, Ptr: OutlinedFn.getArg(i: 1));
2617 OutlinedFn.getArg(i: 1)->replaceUsesWithIf(
2618 New: SharedsOutlined,
2619 ShouldReplace: [SharedsOutlined](Use &U) { return U.getUser() != SharedsOutlined; });
2620
2621 Value *IV = CLI->getIndVar();
2622 Type *IVTy = IV->getType();
2623 Constant *One = ConstantInt::get(Ty: Builder.getInt64Ty(), V: 1);
2624
2625 // When outlining, CodeExtractor will create GEP's to the LowerBound and
2626 // UpperBound. These GEP's can be reused for loading the tasks respective
2627 // bounds.
2628 Value *TaskLB = nullptr;
2629 Value *TaskUB = nullptr;
2630 Value *TaskStep = nullptr;
2631 Value *LoadTaskLB = nullptr;
2632 Value *LoadTaskUB = nullptr;
2633 Value *LoadTaskStep = nullptr;
2634 for (Instruction &I : *TaskloopAllocaBB) {
2635 if (I.getOpcode() == Instruction::GetElementPtr) {
2636 GetElementPtrInst &Gep = cast<GetElementPtrInst>(Val&: I);
2637 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: Gep.getOperand(i_nocapture: 2))) {
2638 switch (CI->getZExtValue()) {
2639 case 0:
2640 TaskLB = &I;
2641 break;
2642 case 1:
2643 TaskUB = &I;
2644 break;
2645 case 2:
2646 TaskStep = &I;
2647 break;
2648 }
2649 }
2650 } else if (I.getOpcode() == Instruction::Load) {
2651 LoadInst &Load = cast<LoadInst>(Val&: I);
2652 if (Load.getPointerOperand() == TaskLB) {
2653 assert(TaskLB != nullptr && "Expected value for TaskLB");
2654 LoadTaskLB = &I;
2655 } else if (Load.getPointerOperand() == TaskUB) {
2656 assert(TaskUB != nullptr && "Expected value for TaskUB");
2657 LoadTaskUB = &I;
2658 } else if (Load.getPointerOperand() == TaskStep) {
2659 assert(TaskStep != nullptr && "Expected value for TaskStep");
2660 LoadTaskStep = &I;
2661 }
2662 }
2663 }
2664
2665 Builder.SetInsertPoint(CLI->getPreheader()->getTerminator());
2666
2667 assert(LoadTaskLB != nullptr && "Expected value for LoadTaskLB");
2668 assert(LoadTaskUB != nullptr && "Expected value for LoadTaskUB");
2669 assert(LoadTaskStep != nullptr && "Expected value for LoadTaskStep");
2670 Value *TripCountMinusOne = Builder.CreateSDiv(
2671 LHS: Builder.CreateSub(LHS: LoadTaskUB, RHS: LoadTaskLB), RHS: LoadTaskStep);
2672 Value *TripCount = Builder.CreateAdd(LHS: TripCountMinusOne, RHS: One, Name: "trip_cnt");
2673 Value *CastedTripCount = Builder.CreateIntCast(V: TripCount, DestTy: IVTy, isSigned: true);
2674 Value *CastedTaskLB = Builder.CreateIntCast(V: LoadTaskLB, DestTy: IVTy, isSigned: true);
2675 // set the trip count in the CLI
2676 CLI->setTripCount(CastedTripCount);
2677
2678 Builder.SetInsertPoint(TheBB: CLI->getBody(),
2679 IP: CLI->getBody()->getFirstInsertionPt());
2680
2681 if (NumOfCollapseLoops > 1) {
2682 llvm::SmallVector<User *> UsersToReplace;
2683 // When using the collapse clause, the bounds of the loop have to be
2684 // adjusted to properly represent the iterator of the outer loop.
2685 Value *IVPlusTaskLB = Builder.CreateAdd(
2686 LHS: CLI->getIndVar(),
2687 RHS: Builder.CreateSub(LHS: CastedTaskLB, RHS: ConstantInt::get(Ty: IVTy, V: 1)));
2688 // To ensure every Use is correctly captured, we first want to record
2689 // which users to replace the value in, and then replace the value.
2690 for (auto IVUse = CLI->getIndVar()->uses().begin();
2691 IVUse != CLI->getIndVar()->uses().end(); IVUse++) {
2692 User *IVUser = IVUse->getUser();
2693 if (auto *Op = dyn_cast<BinaryOperator>(Val: IVUser)) {
2694 if (Op->getOpcode() == Instruction::URem ||
2695 Op->getOpcode() == Instruction::UDiv) {
2696 UsersToReplace.push_back(Elt: IVUser);
2697 }
2698 }
2699 }
2700 for (User *User : UsersToReplace) {
2701 User->replaceUsesOfWith(From: CLI->getIndVar(), To: IVPlusTaskLB);
2702 }
2703 } else {
2704 // The canonical loop is generated with a fixed lower bound. We need to
2705 // update the index calculation code to use the task's lower bound. The
2706 // generated code looks like this:
2707 // %omp_loop.iv = phi ...
2708 // ...
2709 // %tmp = mul [type] %omp_loop.iv, step
2710 // %user_index = add [type] tmp, lb
2711 // OpenMPIRBuilder constructs canonical loops to have exactly three uses
2712 // of the normalised induction variable:
2713 // 1. This one: converting the normalised IV to the user IV
2714 // 2. The increment (add)
2715 // 3. The comparison against the trip count (icmp)
2716 // (1) is the only use that is a mul followed by an add so this cannot
2717 // match other IR.
2718 assert(CLI->getIndVar()->getNumUses() == 3 &&
2719 "Canonical loop should have exactly three uses of the ind var");
2720 for (User *IVUser : CLI->getIndVar()->users()) {
2721 if (auto *Mul = dyn_cast<BinaryOperator>(Val: IVUser)) {
2722 if (Mul->getOpcode() == Instruction::Mul) {
2723 for (User *MulUser : Mul->users()) {
2724 if (auto *Add = dyn_cast<BinaryOperator>(Val: MulUser)) {
2725 if (Add->getOpcode() == Instruction::Add) {
2726 Add->setOperand(i_nocapture: 1, Val_nocapture: CastedTaskLB);
2727 }
2728 }
2729 }
2730 }
2731 }
2732 }
2733 }
2734
2735 FakeLB->replaceAllUsesWith(V: CastedLBVal);
2736 FakeUB->replaceAllUsesWith(V: CastedUBVal);
2737 FakeStep->replaceAllUsesWith(V: CastedStepVal);
2738 for (Instruction *I : llvm::reverse(C&: ToBeDeleted)) {
2739 I->eraseFromParent();
2740 }
2741 };
2742
2743 addOutlineInfo(OI: std::move(OI));
2744 Builder.SetInsertPoint(TheBB: TaskloopExitBB, IP: TaskloopExitBB->begin());
2745 return Builder.saveIP();
2746}
2747
2748llvm::StructType *OpenMPIRBuilder::getKmpTaskAffinityInfoTy() {
2749 llvm::Type *IntPtrTy = llvm::Type::getIntNTy(
2750 C&: M.getContext(), N: M.getDataLayout().getPointerSizeInBits());
2751 return llvm::StructType::get(elt1: IntPtrTy, elts: IntPtrTy,
2752 elts: llvm::Type::getInt32Ty(C&: M.getContext()));
2753}
2754
2755OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTask(
2756 const LocationDescription &Loc, InsertPointTy AllocaIP,
2757 ArrayRef<BasicBlock *> DeallocBlocks, BodyGenCallbackTy BodyGenCB,
2758 bool Tied, Value *Final, Value *IfCondition,
2759 const DependenciesInfo &Dependencies, const AffinityData &Affinities,
2760 bool Mergeable, Value *EventHandle, Value *Priority) {
2761
2762 if (!updateToLocation(Loc))
2763 return InsertPointTy();
2764
2765 uint32_t SrcLocStrSize;
2766 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2767 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2768 // The current basic block is split into four basic blocks. After outlining,
2769 // they will be mapped as follows:
2770 // ```
2771 // def current_fn() {
2772 // current_basic_block:
2773 // br label %task.exit
2774 // task.exit:
2775 // ; instructions after task
2776 // }
2777 // def outlined_fn() {
2778 // task.alloca:
2779 // br label %task.body
2780 // task.body:
2781 // ret void
2782 // }
2783 // ```
2784 BasicBlock *TaskExitBB = splitBB(Builder, /*CreateBranch=*/true, Name: "task.exit");
2785 BasicBlock *TaskBodyBB = splitBB(Builder, /*CreateBranch=*/true, Name: "task.body");
2786 BasicBlock *TaskAllocaBB =
2787 splitBB(Builder, /*CreateBranch=*/true, Name: "task.alloca");
2788
2789 InsertPointTy TaskAllocaIP =
2790 InsertPointTy(TaskAllocaBB, TaskAllocaBB->begin());
2791 InsertPointTy TaskBodyIP = InsertPointTy(TaskBodyBB, TaskBodyBB->begin());
2792 if (Error Err = BodyGenCB(TaskAllocaIP, TaskBodyIP, TaskExitBB))
2793 return Err;
2794
2795 auto OI = std::make_unique<OutlineInfo>();
2796 OI->EntryBB = TaskAllocaBB;
2797 OI->OuterAllocBB = AllocaIP.getBlock();
2798 OI->ExitBB = TaskExitBB;
2799 OI->OuterDeallocBBs.reserve(N: DeallocBlocks.size());
2800 copy(Range&: DeallocBlocks, Out: OI->OuterDeallocBBs.end());
2801
2802 // Add the thread ID argument.
2803 SmallVector<Instruction *, 4> ToBeDeleted;
2804 OI->ExcludeArgsFromAggregate.push_back(Elt: createFakeIntVal(
2805 Builder, OuterAllocaIP: AllocaIP, ToBeDeleted, InnerAllocaIP: TaskAllocaIP, Name: "global.tid", AsPtr: false));
2806
2807 OI->PostOutlineCB = [this, Ident, Tied, Final, IfCondition, Dependencies,
2808 Affinities, Mergeable, Priority, EventHandle,
2809 TaskAllocaBB,
2810 ToBeDeleted](Function &OutlinedFn) mutable {
2811 // Replace the Stale CI by appropriate RTL function call.
2812 assert(OutlinedFn.hasOneUse() &&
2813 "there must be a single user for the outlined function");
2814 CallInst *StaleCI = cast<CallInst>(Val: OutlinedFn.user_back());
2815
2816 // HasShareds is true if any variables are captured in the outlined region,
2817 // false otherwise.
2818 bool HasShareds = StaleCI->arg_size() > 1;
2819 Builder.SetInsertPoint(StaleCI);
2820
2821 // Gather the arguments for emitting the runtime call for
2822 // @__kmpc_omp_task_alloc
2823 Function *TaskAllocFn =
2824 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_alloc);
2825
2826 // Arguments - `loc_ref` (Ident) and `gtid` (ThreadID)
2827 // call.
2828 Value *ThreadID = getOrCreateThreadID(Ident);
2829
2830 // Argument - `flags`
2831 // Task is tied iff (Flags & 1) == 1.
2832 // Task is untied iff (Flags & 1) == 0.
2833 // Task is final iff (Flags & 2) == 2.
2834 // Task is not final iff (Flags & 2) == 0.
2835 // Task is mergeable or merged-if0 iff (Flags & 4) == 4.
2836 // Task is neither mergeable nor merged-if0 iff (Flags & 4) == 0.
2837 // Task is detachable iff (Flags & 64) == 64.
2838 // Task is not detachable iff (Flags & 64) == 0.
2839 // Task is priority iff (Flags & 32) == 32.
2840 // Task is not priority iff (Flags & 32) == 0.
2841 // TODO: Handle the other flags.
2842 Value *Flags = Builder.getInt32(C: Tied);
2843 auto *ConstIfCondition = dyn_cast_or_null<ConstantInt>(Val: IfCondition);
2844 bool UseMergedIf0Path = ConstIfCondition && ConstIfCondition->isZero();
2845 if (Final) {
2846 Value *FinalFlag =
2847 Builder.CreateSelect(C: Final, True: Builder.getInt32(C: 2), False: Builder.getInt32(C: 0));
2848 Flags = Builder.CreateOr(LHS: FinalFlag, RHS: Flags);
2849 }
2850
2851 if (Mergeable || UseMergedIf0Path)
2852 Flags = Builder.CreateOr(LHS: Builder.getInt32(C: 4), RHS: Flags);
2853 if (EventHandle)
2854 Flags = Builder.CreateOr(LHS: Builder.getInt32(C: 64), RHS: Flags);
2855 if (Priority)
2856 Flags = Builder.CreateOr(LHS: Builder.getInt32(C: 32), RHS: Flags);
2857
2858 // Argument - `sizeof_kmp_task_t` (TaskSize)
2859 // Tasksize refers to the size in bytes of kmp_task_t data structure
2860 // including private vars accessed in task.
2861 // TODO: add kmp_task_t_with_privates (privates)
2862 Value *TaskSize = Builder.getInt64(
2863 C: divideCeil(Numerator: M.getDataLayout().getTypeSizeInBits(Ty: Task), Denominator: 8));
2864
2865 // Argument - `sizeof_shareds` (SharedsSize)
2866 // SharedsSize refers to the shareds array size in the kmp_task_t data
2867 // structure.
2868 Value *SharedsSize = Builder.getInt64(C: 0);
2869 if (HasShareds) {
2870 AllocaInst *ArgStructAlloca =
2871 dyn_cast<AllocaInst>(Val: StaleCI->getArgOperand(i: 1));
2872 assert(ArgStructAlloca &&
2873 "Unable to find the alloca instruction corresponding to arguments "
2874 "for extracted function");
2875 std::optional<TypeSize> ArgAllocSize =
2876 ArgStructAlloca->getAllocationSize(DL: M.getDataLayout());
2877 assert(ArgAllocSize &&
2878 "Unable to determine size of arguments for extracted function");
2879 SharedsSize = Builder.getInt64(C: ArgAllocSize->getFixedValue());
2880 }
2881 // Emit the @__kmpc_omp_task_alloc runtime call
2882 // The runtime call returns a pointer to an area where the task captured
2883 // variables must be copied before the task is run (TaskData)
2884 CallInst *TaskData = createRuntimeFunctionCall(
2885 Callee: TaskAllocFn, Args: {/*loc_ref=*/Ident, /*gtid=*/ThreadID, /*flags=*/Flags,
2886 /*sizeof_task=*/TaskSize, /*sizeof_shared=*/SharedsSize,
2887 /*task_func=*/&OutlinedFn});
2888
2889 if (Affinities.Count && Affinities.Info) {
2890 Function *RegAffFn = getOrCreateRuntimeFunctionPtr(
2891 FnID: OMPRTL___kmpc_omp_reg_task_with_affinity);
2892
2893 createRuntimeFunctionCall(Callee: RegAffFn, Args: {Ident, ThreadID, TaskData,
2894 Affinities.Count, Affinities.Info});
2895 }
2896
2897 // Emit detach clause initialization.
2898 // evt = (typeof(evt))__kmpc_task_allow_completion_event(loc, tid,
2899 // task_descriptor);
2900 if (EventHandle) {
2901 Function *TaskDetachFn = getOrCreateRuntimeFunctionPtr(
2902 FnID: OMPRTL___kmpc_task_allow_completion_event);
2903 llvm::Value *EventVal =
2904 createRuntimeFunctionCall(Callee: TaskDetachFn, Args: {Ident, ThreadID, TaskData});
2905 llvm::Value *EventHandleAddr =
2906 Builder.CreatePointerBitCastOrAddrSpaceCast(V: EventHandle,
2907 DestTy: Builder.getPtrTy(AddrSpace: 0));
2908 EventVal = Builder.CreatePtrToInt(V: EventVal, DestTy: Builder.getInt64Ty());
2909 Builder.CreateStore(Val: EventVal, Ptr: EventHandleAddr);
2910 }
2911 // Copy the arguments for outlined function
2912 if (HasShareds) {
2913 Value *Shareds = StaleCI->getArgOperand(i: 1);
2914 Align Alignment = TaskData->getPointerAlignment(DL: M.getDataLayout());
2915 Value *TaskShareds = Builder.CreateLoad(Ty: VoidPtr, Ptr: TaskData);
2916 Builder.CreateMemCpy(Dst: TaskShareds, DstAlign: Alignment, Src: Shareds, SrcAlign: Alignment,
2917 Size: SharedsSize);
2918 }
2919
2920 if (Priority) {
2921 //
2922 // The return type of "__kmpc_omp_task_alloc" is "kmp_task_t *",
2923 // we populate the priority information into the "kmp_task_t" here
2924 //
2925 // The struct "kmp_task_t" definition is available in kmp.h
2926 // kmp_task_t = { shareds, routine, part_id, data1, data2 }
2927 // data2 is used for priority
2928 //
2929 Type *Int32Ty = Builder.getInt32Ty();
2930 Constant *Zero = ConstantInt::get(Ty: Int32Ty, V: 0);
2931 // kmp_task_t* => { ptr }
2932 Type *TaskPtr = StructType::get(elt1: VoidPtr);
2933 Value *TaskGEP =
2934 Builder.CreateInBoundsGEP(Ty: TaskPtr, Ptr: TaskData, IdxList: {Zero, Zero});
2935 // kmp_task_t => { ptr, ptr, i32, ptr, ptr }
2936 Type *TaskStructType = StructType::get(
2937 elt1: VoidPtr, elts: VoidPtr, elts: Builder.getInt32Ty(), elts: VoidPtr, elts: VoidPtr);
2938 Value *PriorityData = Builder.CreateInBoundsGEP(
2939 Ty: TaskStructType, Ptr: TaskGEP, IdxList: {Zero, ConstantInt::get(Ty: Int32Ty, V: 4)});
2940 // kmp_cmplrdata_t => { ptr, ptr }
2941 Type *CmplrStructType = StructType::get(elt1: VoidPtr, elts: VoidPtr);
2942 Value *CmplrData = Builder.CreateInBoundsGEP(Ty: CmplrStructType,
2943 Ptr: PriorityData, IdxList: {Zero, Zero});
2944 Builder.CreateStore(Val: Priority, Ptr: CmplrData);
2945 }
2946
2947 Value *DepArray = nullptr;
2948 Value *NumDeps = nullptr;
2949 if (Dependencies.DepArray) {
2950 DepArray = Dependencies.DepArray;
2951 NumDeps = Dependencies.NumDeps;
2952 } else if (!Dependencies.Deps.empty()) {
2953 DepArray = emitTaskDependencies(OMPBuilder&: *this, Dependencies: Dependencies.Deps);
2954 NumDeps = Builder.getInt32(C: Dependencies.Deps.size());
2955 }
2956
2957 // In the presence of the `if` clause, the following IR is generated:
2958 // ...
2959 // %data = call @__kmpc_omp_task_alloc(...)
2960 // br i1 %if_condition, label %then, label %else
2961 // then:
2962 // call @__kmpc_omp_task(...)
2963 // br label %exit
2964 // else:
2965 // ;; Wait for resolution of dependencies, if any, before
2966 // ;; beginning the task
2967 // call @__kmpc_omp_wait_deps(...)
2968 // call @__kmpc_omp_task_begin_if0(...)
2969 // call @outlined_fn(...)
2970 // call @__kmpc_omp_task_complete_if0(...)
2971 // br label %exit
2972 // exit:
2973 // ...
2974 if (IfCondition && !UseMergedIf0Path) {
2975 // `SplitBlockAndInsertIfThenElse` requires the block to have a
2976 // terminator.
2977 splitBB(Builder, /*CreateBranch=*/true, Name: "if.end");
2978 Instruction *IfTerminator =
2979 Builder.GetInsertPoint()->getParent()->getTerminator();
2980 Instruction *ThenTI = IfTerminator, *ElseTI = nullptr;
2981 Builder.SetInsertPoint(IfTerminator);
2982 SplitBlockAndInsertIfThenElse(Cond: IfCondition, SplitBefore: IfTerminator, ThenTerm: &ThenTI,
2983 ElseTerm: &ElseTI);
2984 Builder.SetInsertPoint(ElseTI);
2985
2986 if (DepArray) {
2987 Function *TaskWaitFn =
2988 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_wait_deps);
2989 createRuntimeFunctionCall(
2990 Callee: TaskWaitFn,
2991 Args: {Ident, ThreadID, NumDeps, DepArray,
2992 ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0),
2993 ConstantPointerNull::get(T: PointerType::getUnqual(C&: M.getContext()))});
2994 }
2995 Function *TaskBeginFn =
2996 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_begin_if0);
2997 Function *TaskCompleteFn =
2998 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_complete_if0);
2999 createRuntimeFunctionCall(Callee: TaskBeginFn, Args: {Ident, ThreadID, TaskData});
3000 CallInst *CI = nullptr;
3001 if (HasShareds)
3002 CI = createRuntimeFunctionCall(Callee: &OutlinedFn, Args: {ThreadID, TaskData});
3003 else
3004 CI = createRuntimeFunctionCall(Callee: &OutlinedFn, Args: {ThreadID});
3005 CI->setDebugLoc(StaleCI->getDebugLoc());
3006 createRuntimeFunctionCall(Callee: TaskCompleteFn, Args: {Ident, ThreadID, TaskData});
3007 Builder.SetInsertPoint(ThenTI);
3008 }
3009
3010 if (DepArray) {
3011 Function *TaskFn =
3012 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_with_deps);
3013 createRuntimeFunctionCall(
3014 Callee: TaskFn,
3015 Args: {Ident, ThreadID, TaskData, NumDeps, DepArray,
3016 ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0),
3017 ConstantPointerNull::get(T: PointerType::getUnqual(C&: M.getContext()))});
3018
3019 } else {
3020 // Emit the @__kmpc_omp_task runtime call to spawn the task
3021 Function *TaskFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task);
3022 createRuntimeFunctionCall(Callee: TaskFn, Args: {Ident, ThreadID, TaskData});
3023 }
3024
3025 StaleCI->eraseFromParent();
3026
3027 Builder.SetInsertPoint(TheBB: TaskAllocaBB, IP: TaskAllocaBB->begin());
3028 if (HasShareds) {
3029 LoadInst *Shareds = Builder.CreateLoad(Ty: VoidPtr, Ptr: OutlinedFn.getArg(i: 1));
3030 OutlinedFn.getArg(i: 1)->replaceUsesWithIf(
3031 New: Shareds, ShouldReplace: [Shareds](Use &U) { return U.getUser() != Shareds; });
3032 }
3033
3034 for (Instruction *I : llvm::reverse(C&: ToBeDeleted))
3035 I->eraseFromParent();
3036 };
3037
3038 addOutlineInfo(OI: std::move(OI));
3039 Builder.SetInsertPoint(TheBB: TaskExitBB, IP: TaskExitBB->begin());
3040
3041 return Builder.saveIP();
3042}
3043
3044OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTaskgroup(
3045 const LocationDescription &Loc, InsertPointTy AllocaIP,
3046 ArrayRef<BasicBlock *> DeallocBlocks, BodyGenCallbackTy BodyGenCB) {
3047 if (!updateToLocation(Loc))
3048 return InsertPointTy();
3049
3050 uint32_t SrcLocStrSize;
3051 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
3052 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
3053 Value *ThreadID = getOrCreateThreadID(Ident);
3054
3055 // Emit the @__kmpc_taskgroup runtime call to start the taskgroup
3056 Function *TaskgroupFn =
3057 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_taskgroup);
3058 createRuntimeFunctionCall(Callee: TaskgroupFn, Args: {Ident, ThreadID});
3059
3060 BasicBlock *TaskgroupExitBB = splitBB(Builder, CreateBranch: true, Name: "taskgroup.exit");
3061 if (Error Err = BodyGenCB(AllocaIP, Builder.saveIP(), DeallocBlocks))
3062 return Err;
3063
3064 Builder.SetInsertPoint(TaskgroupExitBB);
3065 // Emit the @__kmpc_end_taskgroup runtime call to end the taskgroup
3066 Function *EndTaskgroupFn =
3067 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_end_taskgroup);
3068 createRuntimeFunctionCall(Callee: EndTaskgroupFn, Args: {Ident, ThreadID});
3069
3070 return Builder.saveIP();
3071}
3072
3073OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createSections(
3074 const LocationDescription &Loc, InsertPointTy AllocaIP,
3075 ArrayRef<StorableBodyGenCallbackTy> SectionCBs, PrivatizeCallbackTy PrivCB,
3076 FinalizeCallbackTy FiniCB, bool IsCancellable, bool IsNowait) {
3077 assert(!isConflictIP(AllocaIP, Loc.IP) && "Dedicated IP allocas required");
3078
3079 if (!updateToLocation(Loc))
3080 return Loc.IP;
3081
3082 FinalizationStack.push_back(Elt: {FiniCB, OMPD_sections, IsCancellable});
3083
3084 // Each section is emitted as a switch case
3085 // Each finalization callback is handled from clang.EmitOMPSectionDirective()
3086 // -> OMP.createSection() which generates the IR for each section
3087 // Iterate through all sections and emit a switch construct:
3088 // switch (IV) {
3089 // case 0:
3090 // <SectionStmt[0]>;
3091 // break;
3092 // ...
3093 // case <NumSection> - 1:
3094 // <SectionStmt[<NumSection> - 1]>;
3095 // break;
3096 // }
3097 // ...
3098 // section_loop.after:
3099 // <FiniCB>;
3100 auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, Value *IndVar) -> Error {
3101 Builder.restoreIP(IP: CodeGenIP);
3102 BasicBlock *Continue =
3103 splitBBWithSuffix(Builder, /*CreateBranch=*/false, Suffix: ".sections.after");
3104 Function *CurFn = Continue->getParent();
3105 SwitchInst *SwitchStmt = Builder.CreateSwitch(V: IndVar, Dest: Continue);
3106
3107 unsigned CaseNumber = 0;
3108 for (auto SectionCB : SectionCBs) {
3109 BasicBlock *CaseBB = BasicBlock::Create(
3110 Context&: M.getContext(), Name: "omp_section_loop.body.case", Parent: CurFn, InsertBefore: Continue);
3111 SwitchStmt->addCase(OnVal: Builder.getInt32(C: CaseNumber), Dest: CaseBB);
3112 Builder.SetInsertPoint(CaseBB);
3113 UncondBrInst *CaseEndBr = Builder.CreateBr(Dest: Continue);
3114 if (Error Err =
3115 SectionCB(InsertPointTy(),
3116 {CaseEndBr->getParent(), CaseEndBr->getIterator()}, {}))
3117 return Err;
3118 CaseNumber++;
3119 }
3120 // remove the existing terminator from body BB since there can be no
3121 // terminators after switch/case
3122 return Error::success();
3123 };
3124 // Loop body ends here
3125 // LowerBound, UpperBound, and STride for createCanonicalLoop
3126 Type *I32Ty = Type::getInt32Ty(C&: M.getContext());
3127 Value *LB = ConstantInt::get(Ty: I32Ty, V: 0);
3128 Value *UB = ConstantInt::get(Ty: I32Ty, V: SectionCBs.size());
3129 Value *ST = ConstantInt::get(Ty: I32Ty, V: 1);
3130 Expected<CanonicalLoopInfo *> LoopInfo = createCanonicalLoop(
3131 Loc, BodyGenCB: LoopBodyGenCB, Start: LB, Stop: UB, Step: ST, IsSigned: true, InclusiveStop: false, ComputeIP: AllocaIP, Name: "section_loop");
3132 if (!LoopInfo)
3133 return LoopInfo.takeError();
3134
3135 InsertPointOrErrorTy WsloopIP =
3136 applyStaticWorkshareLoop(DL: Loc.DL, CLI: *LoopInfo, AllocaIP,
3137 LoopType: WorksharingLoopType::ForStaticLoop, NeedsBarrier: !IsNowait);
3138 if (!WsloopIP)
3139 return WsloopIP.takeError();
3140 InsertPointTy AfterIP = *WsloopIP;
3141
3142 BasicBlock *LoopFini = AfterIP.getBlock()->getSinglePredecessor();
3143 assert(LoopFini && "Bad structure of static workshare loop finalization");
3144
3145 // Apply the finalization callback in LoopAfterBB
3146 auto FiniInfo = FinalizationStack.pop_back_val();
3147 assert(FiniInfo.DK == OMPD_sections &&
3148 "Unexpected finalization stack state!");
3149 if (Error Err = FiniInfo.mergeFiniBB(Builder, OtherFiniBB: LoopFini))
3150 return Err;
3151
3152 return AfterIP;
3153}
3154
3155OpenMPIRBuilder::InsertPointOrErrorTy
3156OpenMPIRBuilder::createSection(const LocationDescription &Loc,
3157 BodyGenCallbackTy BodyGenCB,
3158 FinalizeCallbackTy FiniCB) {
3159 if (!updateToLocation(Loc))
3160 return Loc.IP;
3161
3162 auto FiniCBWrapper = [&](InsertPointTy IP) {
3163 if (IP.getBlock()->end() != IP.getPoint())
3164 return FiniCB(IP);
3165 // This must be done otherwise any nested constructs using FinalizeOMPRegion
3166 // will fail because that function requires the Finalization Basic Block to
3167 // have a terminator, which is already removed by EmitOMPRegionBody.
3168 // IP is currently at cancelation block.
3169 // We need to backtrack to the condition block to fetch
3170 // the exit block and create a branch from cancelation
3171 // to exit block.
3172 IRBuilder<>::InsertPointGuard IPG(Builder);
3173 Builder.restoreIP(IP);
3174 auto *CaseBB = Loc.IP.getBlock();
3175 auto *CondBB = CaseBB->getSinglePredecessor()->getSinglePredecessor();
3176 auto *ExitBB = CondBB->getTerminator()->getSuccessor(Idx: 1);
3177 Instruction *I = Builder.CreateBr(Dest: ExitBB);
3178 IP = InsertPointTy(I->getParent(), I->getIterator());
3179 return FiniCB(IP);
3180 };
3181
3182 Directive OMPD = Directive::OMPD_sections;
3183 // Since we are using Finalization Callback here, HasFinalize
3184 // and IsCancellable have to be true
3185 return EmitOMPInlinedRegion(OMPD, EntryCall: nullptr, ExitCall: nullptr, BodyGenCB, FiniCB: FiniCBWrapper,
3186 /*Conditional*/ false, /*hasFinalize*/ HasFinalize: true,
3187 /*IsCancellable*/ true);
3188}
3189
3190static OpenMPIRBuilder::InsertPointTy getInsertPointAfterInstr(Instruction *I) {
3191 BasicBlock::iterator IT(I);
3192 IT++;
3193 return OpenMPIRBuilder::InsertPointTy(I->getParent(), IT);
3194}
3195
3196Value *OpenMPIRBuilder::getGPUThreadID() {
3197 return createRuntimeFunctionCall(
3198 Callee: getOrCreateRuntimeFunction(M,
3199 FnID: OMPRTL___kmpc_get_hardware_thread_id_in_block),
3200 Args: {});
3201}
3202
3203Value *OpenMPIRBuilder::getGPUWarpSize() {
3204 return createRuntimeFunctionCall(
3205 Callee: getOrCreateRuntimeFunction(M, FnID: OMPRTL___kmpc_get_warp_size), Args: {});
3206}
3207
3208Value *OpenMPIRBuilder::getNVPTXWarpID() {
3209 unsigned LaneIDBits = Log2_32(Value: Config.getGridValue().GV_Warp_Size);
3210 return Builder.CreateAShr(LHS: getGPUThreadID(), RHS: LaneIDBits, Name: "nvptx_warp_id");
3211}
3212
3213Value *OpenMPIRBuilder::getNVPTXLaneID() {
3214 unsigned LaneIDBits = Log2_32(Value: Config.getGridValue().GV_Warp_Size);
3215 assert(LaneIDBits < 32 && "Invalid LaneIDBits size in NVPTX device.");
3216 unsigned LaneIDMask = ~0u >> (32u - LaneIDBits);
3217 return Builder.CreateAnd(LHS: getGPUThreadID(), RHS: Builder.getInt32(C: LaneIDMask),
3218 Name: "nvptx_lane_id");
3219}
3220
3221Value *OpenMPIRBuilder::castValueToType(InsertPointTy AllocaIP, Value *From,
3222 Type *ToType) {
3223 Type *FromType = From->getType();
3224 uint64_t FromSize = M.getDataLayout().getTypeStoreSize(Ty: FromType);
3225 uint64_t ToSize = M.getDataLayout().getTypeStoreSize(Ty: ToType);
3226 assert(FromSize > 0 && "From size must be greater than zero");
3227 assert(ToSize > 0 && "To size must be greater than zero");
3228 if (FromType == ToType)
3229 return From;
3230 if (FromSize == ToSize)
3231 return Builder.CreateBitCast(V: From, DestTy: ToType);
3232 if (ToType->isIntegerTy() && FromType->isIntegerTy())
3233 return Builder.CreateIntCast(V: From, DestTy: ToType, /*isSigned*/ true);
3234 InsertPointTy SaveIP = Builder.saveIP();
3235 Builder.restoreIP(IP: AllocaIP);
3236 Value *CastItem = Builder.CreateAlloca(Ty: ToType);
3237 Builder.restoreIP(IP: SaveIP);
3238
3239 Value *ValCastItem = Builder.CreatePointerBitCastOrAddrSpaceCast(
3240 V: CastItem, DestTy: Builder.getPtrTy(AddrSpace: 0));
3241 Builder.CreateStore(Val: From, Ptr: ValCastItem);
3242 return Builder.CreateLoad(Ty: ToType, Ptr: CastItem);
3243}
3244
3245Value *OpenMPIRBuilder::createRuntimeShuffleFunction(InsertPointTy AllocaIP,
3246 Value *Element,
3247 Type *ElementType,
3248 Value *Offset) {
3249 uint64_t Size = M.getDataLayout().getTypeStoreSize(Ty: ElementType);
3250 assert(Size <= 8 && "Unsupported bitwidth in shuffle instruction");
3251
3252 // Cast all types to 32- or 64-bit values before calling shuffle routines.
3253 Type *CastTy = Builder.getIntNTy(N: Size <= 4 ? 32 : 64);
3254 Value *ElemCast = castValueToType(AllocaIP, From: Element, ToType: CastTy);
3255 Value *WarpSize =
3256 Builder.CreateIntCast(V: getGPUWarpSize(), DestTy: Builder.getInt16Ty(), isSigned: true);
3257 Function *ShuffleFunc = getOrCreateRuntimeFunctionPtr(
3258 FnID: Size <= 4 ? RuntimeFunction::OMPRTL___kmpc_shuffle_int32
3259 : RuntimeFunction::OMPRTL___kmpc_shuffle_int64);
3260 Value *WarpSizeCast =
3261 Builder.CreateIntCast(V: WarpSize, DestTy: Builder.getInt16Ty(), /*isSigned=*/true);
3262 Value *ShuffleCall =
3263 createRuntimeFunctionCall(Callee: ShuffleFunc, Args: {ElemCast, Offset, WarpSizeCast});
3264 return castValueToType(AllocaIP, From: ShuffleCall, ToType: CastTy);
3265}
3266
3267void OpenMPIRBuilder::shuffleAndStore(InsertPointTy AllocaIP, Value *SrcAddr,
3268 Value *DstAddr, Type *ElemType,
3269 Value *Offset, Type *ReductionArrayTy,
3270 bool IsByRefElem) {
3271 uint64_t Size = M.getDataLayout().getTypeStoreSize(Ty: ElemType);
3272 // Create the loop over the big sized data.
3273 // ptr = (void*)Elem;
3274 // ptrEnd = (void*) Elem + 1;
3275 // Step = 8;
3276 // while (ptr + Step < ptrEnd)
3277 // shuffle((int64_t)*ptr);
3278 // Step = 4;
3279 // while (ptr + Step < ptrEnd)
3280 // shuffle((int32_t)*ptr);
3281 // ...
3282 Type *IndexTy = Builder.getIndexTy(
3283 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
3284 Value *ElemPtr = DstAddr;
3285 Value *Ptr = SrcAddr;
3286 for (unsigned IntSize = 8; IntSize >= 1; IntSize /= 2) {
3287 if (Size < IntSize)
3288 continue;
3289 Type *IntType = Builder.getIntNTy(N: IntSize * 8);
3290 Ptr = Builder.CreatePointerBitCastOrAddrSpaceCast(
3291 V: Ptr, DestTy: Builder.getPtrTy(AddrSpace: 0), Name: Ptr->getName() + ".ascast");
3292 Value *SrcAddrGEP =
3293 Builder.CreateGEP(Ty: ElemType, Ptr: SrcAddr, IdxList: {ConstantInt::get(Ty: IndexTy, V: 1)});
3294 ElemPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
3295 V: ElemPtr, DestTy: Builder.getPtrTy(AddrSpace: 0), Name: ElemPtr->getName() + ".ascast");
3296
3297 Function *CurFunc = Builder.GetInsertBlock()->getParent();
3298 if ((Size / IntSize) > 1) {
3299 Value *PtrEnd = Builder.CreatePointerBitCastOrAddrSpaceCast(
3300 V: SrcAddrGEP, DestTy: Builder.getPtrTy());
3301 BasicBlock *PreCondBB =
3302 BasicBlock::Create(Context&: M.getContext(), Name: ".shuffle.pre_cond");
3303 BasicBlock *ThenBB = BasicBlock::Create(Context&: M.getContext(), Name: ".shuffle.then");
3304 BasicBlock *ExitBB = BasicBlock::Create(Context&: M.getContext(), Name: ".shuffle.exit");
3305 BasicBlock *CurrentBB = Builder.GetInsertBlock();
3306 emitBlock(BB: PreCondBB, CurFn: CurFunc);
3307 PHINode *PhiSrc =
3308 Builder.CreatePHI(Ty: Ptr->getType(), /*NumReservedValues=*/2);
3309 PhiSrc->addIncoming(V: Ptr, BB: CurrentBB);
3310 PHINode *PhiDest =
3311 Builder.CreatePHI(Ty: ElemPtr->getType(), /*NumReservedValues=*/2);
3312 PhiDest->addIncoming(V: ElemPtr, BB: CurrentBB);
3313 Ptr = PhiSrc;
3314 ElemPtr = PhiDest;
3315 Value *PtrDiff = Builder.CreatePtrDiff(
3316 ElemTy: Builder.getInt8Ty(), LHS: PtrEnd,
3317 RHS: Builder.CreatePointerBitCastOrAddrSpaceCast(V: Ptr, DestTy: Builder.getPtrTy()));
3318 Builder.CreateCondBr(
3319 Cond: Builder.CreateICmpSGT(LHS: PtrDiff, RHS: Builder.getInt64(C: IntSize - 1)), True: ThenBB,
3320 False: ExitBB);
3321 emitBlock(BB: ThenBB, CurFn: CurFunc);
3322 Value *Res = createRuntimeShuffleFunction(
3323 AllocaIP,
3324 Element: Builder.CreateAlignedLoad(
3325 Ty: IntType, Ptr, Align: M.getDataLayout().getPrefTypeAlign(Ty: ElemType)),
3326 ElementType: IntType, Offset);
3327 Builder.CreateAlignedStore(Val: Res, Ptr: ElemPtr,
3328 Align: M.getDataLayout().getPrefTypeAlign(Ty: ElemType));
3329 Value *LocalPtr =
3330 Builder.CreateGEP(Ty: IntType, Ptr, IdxList: {ConstantInt::get(Ty: IndexTy, V: 1)});
3331 Value *LocalElemPtr =
3332 Builder.CreateGEP(Ty: IntType, Ptr: ElemPtr, IdxList: {ConstantInt::get(Ty: IndexTy, V: 1)});
3333 PhiSrc->addIncoming(V: LocalPtr, BB: ThenBB);
3334 PhiDest->addIncoming(V: LocalElemPtr, BB: ThenBB);
3335 emitBranch(Target: PreCondBB);
3336 emitBlock(BB: ExitBB, CurFn: CurFunc);
3337 } else {
3338 Value *Res = createRuntimeShuffleFunction(
3339 AllocaIP, Element: Builder.CreateLoad(Ty: IntType, Ptr), ElementType: IntType, Offset);
3340 if (ElemType->isIntegerTy() && ElemType->getScalarSizeInBits() <
3341 Res->getType()->getScalarSizeInBits())
3342 Res = Builder.CreateTrunc(V: Res, DestTy: ElemType);
3343 Builder.CreateStore(Val: Res, Ptr: ElemPtr);
3344 Ptr = Builder.CreateGEP(Ty: IntType, Ptr, IdxList: {ConstantInt::get(Ty: IndexTy, V: 1)});
3345 ElemPtr =
3346 Builder.CreateGEP(Ty: IntType, Ptr: ElemPtr, IdxList: {ConstantInt::get(Ty: IndexTy, V: 1)});
3347 }
3348 Size = Size % IntSize;
3349 }
3350}
3351
3352Error OpenMPIRBuilder::emitReductionListCopy(
3353 InsertPointTy AllocaIP, CopyAction Action, Type *ReductionArrayTy,
3354 ArrayRef<ReductionInfo> ReductionInfos, Value *SrcBase, Value *DestBase,
3355 ArrayRef<bool> IsByRef, CopyOptionsTy CopyOptions) {
3356 Type *IndexTy = Builder.getIndexTy(
3357 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
3358 Value *RemoteLaneOffset = CopyOptions.RemoteLaneOffset;
3359
3360 // Iterates, element-by-element, through the source Reduce list and
3361 // make a copy.
3362 for (auto En : enumerate(First&: ReductionInfos)) {
3363 const ReductionInfo &RI = En.value();
3364 Value *SrcElementAddr = nullptr;
3365 AllocaInst *DestAlloca = nullptr;
3366 Value *DestElementAddr = nullptr;
3367 Value *DestElementPtrAddr = nullptr;
3368 // Should we shuffle in an element from a remote lane?
3369 bool ShuffleInElement = false;
3370 // Set to true to update the pointer in the dest Reduce list to a
3371 // newly created element.
3372 bool UpdateDestListPtr = false;
3373
3374 // Step 1.1: Get the address for the src element in the Reduce list.
3375 Value *SrcElementPtrAddr = Builder.CreateInBoundsGEP(
3376 Ty: ReductionArrayTy, Ptr: SrcBase,
3377 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
3378 SrcElementAddr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: SrcElementPtrAddr);
3379
3380 // Step 1.2: Create a temporary to store the element in the destination
3381 // Reduce list.
3382 DestElementPtrAddr = Builder.CreateInBoundsGEP(
3383 Ty: ReductionArrayTy, Ptr: DestBase,
3384 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
3385 bool IsByRefElem = (!IsByRef.empty() && IsByRef[En.index()]);
3386 switch (Action) {
3387 case CopyAction::RemoteLaneToThread: {
3388 InsertPointTy CurIP = Builder.saveIP();
3389 Builder.restoreIP(IP: AllocaIP);
3390
3391 Type *DestAllocaType =
3392 IsByRefElem ? RI.ByRefAllocatedType : RI.ElementType;
3393 DestAlloca = Builder.CreateAlloca(Ty: DestAllocaType, ArraySize: nullptr,
3394 Name: ".omp.reduction.element");
3395 DestAlloca->setAlignment(
3396 M.getDataLayout().getPrefTypeAlign(Ty: DestAllocaType));
3397 DestElementAddr = DestAlloca;
3398 DestElementAddr =
3399 Builder.CreateAddrSpaceCast(V: DestElementAddr, DestTy: Builder.getPtrTy(),
3400 Name: DestElementAddr->getName() + ".ascast");
3401 Builder.restoreIP(IP: CurIP);
3402 ShuffleInElement = true;
3403 UpdateDestListPtr = true;
3404 break;
3405 }
3406 case CopyAction::ThreadCopy: {
3407 DestElementAddr =
3408 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: DestElementPtrAddr);
3409 break;
3410 }
3411 }
3412
3413 // Now that all active lanes have read the element in the
3414 // Reduce list, shuffle over the value from the remote lane.
3415 if (ShuffleInElement) {
3416 Type *ShuffleType = RI.ElementType;
3417 Value *ShuffleSrcAddr = SrcElementAddr;
3418 Value *ShuffleDestAddr = DestElementAddr;
3419 AllocaInst *LocalStorage = nullptr;
3420
3421 if (IsByRefElem) {
3422 assert(RI.ByRefElementType && "Expected by-ref element type to be set");
3423 assert(RI.ByRefAllocatedType &&
3424 "Expected by-ref allocated type to be set");
3425 // For by-ref reductions, we need to copy from the remote lane the
3426 // actual value of the partial reduction computed by that remote lane;
3427 // rather than, for example, a pointer to that data or, even worse, a
3428 // pointer to the descriptor of the by-ref reduction element.
3429 ShuffleType = RI.ByRefElementType;
3430
3431 if (RI.DataPtrPtrGen) {
3432 // Descriptor-based by-ref: extract data pointer from descriptor.
3433 InsertPointOrErrorTy GenResult = RI.DataPtrPtrGen(
3434 Builder.saveIP(), ShuffleSrcAddr, ShuffleSrcAddr);
3435
3436 if (!GenResult)
3437 return GenResult.takeError();
3438
3439 ShuffleSrcAddr =
3440 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ShuffleSrcAddr);
3441
3442 {
3443 InsertPointTy OldIP = Builder.saveIP();
3444 Builder.restoreIP(IP: AllocaIP);
3445
3446 LocalStorage = Builder.CreateAlloca(Ty: ShuffleType);
3447 Builder.restoreIP(IP: OldIP);
3448 ShuffleDestAddr = LocalStorage;
3449 }
3450 } else {
3451 // Non-descriptor by-ref: the pointer already references data
3452 // directly. Shuffle into the destination alloca.
3453 ShuffleDestAddr = DestElementAddr;
3454 }
3455 }
3456
3457 shuffleAndStore(AllocaIP, SrcAddr: ShuffleSrcAddr, DstAddr: ShuffleDestAddr, ElemType: ShuffleType,
3458 Offset: RemoteLaneOffset, ReductionArrayTy, IsByRefElem);
3459
3460 if (IsByRefElem && RI.DataPtrPtrGen) {
3461 // Copy descriptor from source and update base_ptr to shuffled data
3462 Value *DestDescriptorAddr = Builder.CreatePointerBitCastOrAddrSpaceCast(
3463 V: DestAlloca, DestTy: Builder.getPtrTy(), Name: ".ascast");
3464
3465 InsertPointOrErrorTy GenResult = generateReductionDescriptor(
3466 DescriptorAddr: DestDescriptorAddr, DataPtr: LocalStorage, SrcDescriptorAddr: SrcElementAddr,
3467 DescriptorType: RI.ByRefAllocatedType, DataPtrPtrGen: RI.DataPtrPtrGen);
3468
3469 if (!GenResult)
3470 return GenResult.takeError();
3471 }
3472 } else {
3473 switch (RI.EvaluationKind) {
3474 case EvalKind::Scalar: {
3475 Value *Elem = Builder.CreateLoad(Ty: RI.ElementType, Ptr: SrcElementAddr);
3476 // Store the source element value to the dest element address.
3477 Builder.CreateStore(Val: Elem, Ptr: DestElementAddr);
3478 break;
3479 }
3480 case EvalKind::Complex: {
3481 Value *SrcRealPtr = Builder.CreateConstInBoundsGEP2_32(
3482 Ty: RI.ElementType, Ptr: SrcElementAddr, Idx0: 0, Idx1: 0, Name: ".realp");
3483 Value *SrcReal = Builder.CreateLoad(
3484 Ty: RI.ElementType->getStructElementType(N: 0), Ptr: SrcRealPtr, Name: ".real");
3485 Value *SrcImgPtr = Builder.CreateConstInBoundsGEP2_32(
3486 Ty: RI.ElementType, Ptr: SrcElementAddr, Idx0: 0, Idx1: 1, Name: ".imagp");
3487 Value *SrcImg = Builder.CreateLoad(
3488 Ty: RI.ElementType->getStructElementType(N: 1), Ptr: SrcImgPtr, Name: ".imag");
3489
3490 Value *DestRealPtr = Builder.CreateConstInBoundsGEP2_32(
3491 Ty: RI.ElementType, Ptr: DestElementAddr, Idx0: 0, Idx1: 0, Name: ".realp");
3492 Value *DestImgPtr = Builder.CreateConstInBoundsGEP2_32(
3493 Ty: RI.ElementType, Ptr: DestElementAddr, Idx0: 0, Idx1: 1, Name: ".imagp");
3494 Builder.CreateStore(Val: SrcReal, Ptr: DestRealPtr);
3495 Builder.CreateStore(Val: SrcImg, Ptr: DestImgPtr);
3496 break;
3497 }
3498 case EvalKind::Aggregate: {
3499 Value *SizeVal = Builder.getInt64(
3500 C: M.getDataLayout().getTypeStoreSize(Ty: RI.ElementType));
3501 Builder.CreateMemCpy(
3502 Dst: DestElementAddr, DstAlign: M.getDataLayout().getPrefTypeAlign(Ty: RI.ElementType),
3503 Src: SrcElementAddr, SrcAlign: M.getDataLayout().getPrefTypeAlign(Ty: RI.ElementType),
3504 Size: SizeVal, isVolatile: false);
3505 break;
3506 }
3507 };
3508 }
3509
3510 // Step 3.1: Modify reference in dest Reduce list as needed.
3511 // Modifying the reference in Reduce list to point to the newly
3512 // created element. The element is live in the current function
3513 // scope and that of functions it invokes (i.e., reduce_function).
3514 // RemoteReduceData[i] = (void*)&RemoteElem
3515 if (UpdateDestListPtr) {
3516 Value *CastDestAddr = Builder.CreatePointerBitCastOrAddrSpaceCast(
3517 V: DestElementAddr, DestTy: Builder.getPtrTy(),
3518 Name: DestElementAddr->getName() + ".ascast");
3519 Builder.CreateStore(Val: CastDestAddr, Ptr: DestElementPtrAddr);
3520 }
3521 }
3522
3523 return Error::success();
3524}
3525
3526Expected<Function *> OpenMPIRBuilder::emitInterWarpCopyFunction(
3527 const LocationDescription &Loc, ArrayRef<ReductionInfo> ReductionInfos,
3528 AttributeList FuncAttrs, ArrayRef<bool> IsByRef) {
3529 InsertPointTy SavedIP = Builder.saveIP();
3530 LLVMContext &Ctx = M.getContext();
3531 FunctionType *FuncTy = FunctionType::get(
3532 Result: Builder.getVoidTy(), Params: {Builder.getPtrTy(), Builder.getInt32Ty()},
3533 /* IsVarArg */ isVarArg: false);
3534 Function *WcFunc =
3535 Function::Create(Ty: FuncTy, Linkage: GlobalVariable::InternalLinkage,
3536 N: "_omp_reduction_inter_warp_copy_func", M: &M);
3537 WcFunc->setCallingConv(Config.getRuntimeCC());
3538 WcFunc->setAttributes(FuncAttrs);
3539 WcFunc->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
3540 WcFunc->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
3541 BasicBlock *EntryBB = BasicBlock::Create(Context&: M.getContext(), Name: "entry", Parent: WcFunc);
3542 Builder.SetInsertPoint(EntryBB);
3543
3544 // ReduceList: thread local Reduce list.
3545 // At the stage of the computation when this function is called, partially
3546 // aggregated values reside in the first lane of every active warp.
3547 Argument *ReduceListArg = WcFunc->getArg(i: 0);
3548 // NumWarps: number of warps active in the parallel region. This could
3549 // be smaller than 32 (max warps in a CTA) for partial block reduction.
3550 Argument *NumWarpsArg = WcFunc->getArg(i: 1);
3551
3552 // This array is used as a medium to transfer, one reduce element at a time,
3553 // the data from the first lane of every warp to lanes in the first warp
3554 // in order to perform the final step of a reduction in a parallel region
3555 // (reduction across warps). The array is placed in NVPTX __shared__ memory
3556 // for reduced latency, as well as to have a distinct copy for concurrently
3557 // executing target regions. The array is declared with common linkage so
3558 // as to be shared across compilation units.
3559 StringRef TransferMediumName =
3560 "__openmp_nvptx_data_transfer_temporary_storage";
3561 GlobalVariable *TransferMedium = M.getGlobalVariable(Name: TransferMediumName);
3562 unsigned WarpSize = Config.getGridValue().GV_Warp_Size;
3563 ArrayType *ArrayTy = ArrayType::get(ElementType: Builder.getInt32Ty(), NumElements: WarpSize);
3564 if (!TransferMedium) {
3565 TransferMedium = new GlobalVariable(
3566 M, ArrayTy, /*isConstant=*/false, GlobalVariable::WeakAnyLinkage,
3567 UndefValue::get(T: ArrayTy), TransferMediumName,
3568 /*InsertBefore=*/nullptr, GlobalVariable::NotThreadLocal,
3569 /*AddressSpace=*/3);
3570 }
3571
3572 // Get the CUDA thread id of the current OpenMP thread on the GPU.
3573 Value *GPUThreadID = getGPUThreadID();
3574 // nvptx_lane_id = nvptx_id % warpsize
3575 Value *LaneID = getNVPTXLaneID();
3576 // nvptx_warp_id = nvptx_id / warpsize
3577 Value *WarpID = getNVPTXWarpID();
3578
3579 InsertPointTy AllocaIP =
3580 InsertPointTy(Builder.GetInsertBlock(),
3581 Builder.GetInsertBlock()->getFirstInsertionPt());
3582 Type *Arg0Type = ReduceListArg->getType();
3583 Type *Arg1Type = NumWarpsArg->getType();
3584 Builder.restoreIP(IP: AllocaIP);
3585 AllocaInst *ReduceListAlloca = Builder.CreateAlloca(
3586 Ty: Arg0Type, ArraySize: nullptr, Name: ReduceListArg->getName() + ".addr");
3587 AllocaInst *NumWarpsAlloca =
3588 Builder.CreateAlloca(Ty: Arg1Type, ArraySize: nullptr, Name: NumWarpsArg->getName() + ".addr");
3589 Value *ReduceListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
3590 V: ReduceListAlloca, DestTy: Arg0Type, Name: ReduceListAlloca->getName() + ".ascast");
3591 Value *NumWarpsAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
3592 V: NumWarpsAlloca, DestTy: Builder.getPtrTy(AddrSpace: 0),
3593 Name: NumWarpsAlloca->getName() + ".ascast");
3594 Builder.CreateStore(Val: ReduceListArg, Ptr: ReduceListAddrCast);
3595 Builder.CreateStore(Val: NumWarpsArg, Ptr: NumWarpsAddrCast);
3596 AllocaIP = getInsertPointAfterInstr(I: NumWarpsAlloca);
3597 InsertPointTy CodeGenIP =
3598 getInsertPointAfterInstr(I: &Builder.GetInsertBlock()->back());
3599 Builder.restoreIP(IP: CodeGenIP);
3600
3601 Value *ReduceList =
3602 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ReduceListAddrCast);
3603
3604 for (auto En : enumerate(First&: ReductionInfos)) {
3605 //
3606 // Warp master copies reduce element to transfer medium in __shared__
3607 // memory.
3608 //
3609 const ReductionInfo &RI = En.value();
3610 bool IsByRefElem = !IsByRef.empty() && IsByRef[En.index()];
3611 unsigned RealTySize = M.getDataLayout().getTypeAllocSize(
3612 Ty: IsByRefElem ? RI.ByRefElementType : RI.ElementType);
3613 for (unsigned TySize = 4; TySize > 0 && RealTySize > 0; TySize /= 2) {
3614 Type *CType = Builder.getIntNTy(N: TySize * 8);
3615
3616 unsigned NumIters = RealTySize / TySize;
3617 if (NumIters == 0)
3618 continue;
3619 Value *Cnt = nullptr;
3620 Value *CntAddr = nullptr;
3621 BasicBlock *PrecondBB = nullptr;
3622 BasicBlock *ExitBB = nullptr;
3623 if (NumIters > 1) {
3624 CodeGenIP = Builder.saveIP();
3625 Builder.restoreIP(IP: AllocaIP);
3626 CntAddr =
3627 Builder.CreateAlloca(Ty: Builder.getInt32Ty(), ArraySize: nullptr, Name: ".cnt.addr");
3628
3629 CntAddr = Builder.CreateAddrSpaceCast(V: CntAddr, DestTy: Builder.getPtrTy(),
3630 Name: CntAddr->getName() + ".ascast");
3631 Builder.restoreIP(IP: CodeGenIP);
3632 Builder.CreateStore(Val: Constant::getNullValue(Ty: Builder.getInt32Ty()),
3633 Ptr: CntAddr,
3634 /*Volatile=*/isVolatile: false);
3635 PrecondBB = BasicBlock::Create(Context&: Ctx, Name: "precond");
3636 ExitBB = BasicBlock::Create(Context&: Ctx, Name: "exit");
3637 BasicBlock *BodyBB = BasicBlock::Create(Context&: Ctx, Name: "body");
3638 emitBlock(BB: PrecondBB, CurFn: Builder.GetInsertBlock()->getParent());
3639 Cnt = Builder.CreateLoad(Ty: Builder.getInt32Ty(), Ptr: CntAddr,
3640 /*Volatile=*/isVolatile: false);
3641 Value *Cmp = Builder.CreateICmpULT(
3642 LHS: Cnt, RHS: ConstantInt::get(Ty: Builder.getInt32Ty(), V: NumIters));
3643 Builder.CreateCondBr(Cond: Cmp, True: BodyBB, False: ExitBB);
3644 emitBlock(BB: BodyBB, CurFn: Builder.GetInsertBlock()->getParent());
3645 }
3646
3647 // kmpc_barrier.
3648 InsertPointOrErrorTy BarrierIP1 =
3649 createBarrier(Loc: LocationDescription(Builder.saveIP(), Loc.DL),
3650 Kind: omp::Directive::OMPD_unknown,
3651 /* ForceSimpleCall */ false,
3652 /* CheckCancelFlag */ true);
3653 if (!BarrierIP1)
3654 return BarrierIP1.takeError();
3655 BasicBlock *ThenBB = BasicBlock::Create(Context&: Ctx, Name: "then");
3656 BasicBlock *ElseBB = BasicBlock::Create(Context&: Ctx, Name: "else");
3657 BasicBlock *MergeBB = BasicBlock::Create(Context&: Ctx, Name: "ifcont");
3658
3659 // if (lane_id == 0)
3660 Value *IsWarpMaster = Builder.CreateIsNull(Arg: LaneID, Name: "warp_master");
3661 Builder.CreateCondBr(Cond: IsWarpMaster, True: ThenBB, False: ElseBB);
3662 emitBlock(BB: ThenBB, CurFn: Builder.GetInsertBlock()->getParent());
3663
3664 // Reduce element = LocalReduceList[i]
3665 auto *RedListArrayTy =
3666 ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: ReductionInfos.size());
3667 Type *IndexTy = Builder.getIndexTy(
3668 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
3669 Value *ElemPtrPtr =
3670 Builder.CreateInBoundsGEP(Ty: RedListArrayTy, Ptr: ReduceList,
3671 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0),
3672 ConstantInt::get(Ty: IndexTy, V: En.index())});
3673 // elemptr = ((CopyType*)(elemptrptr)) + I
3674 Value *ElemPtr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ElemPtrPtr);
3675
3676 if (IsByRefElem && RI.DataPtrPtrGen) {
3677 InsertPointOrErrorTy GenRes =
3678 RI.DataPtrPtrGen(Builder.saveIP(), ElemPtr, ElemPtr);
3679
3680 if (!GenRes)
3681 return GenRes.takeError();
3682
3683 ElemPtr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ElemPtr);
3684 }
3685
3686 if (NumIters > 1)
3687 ElemPtr = Builder.CreateGEP(Ty: Builder.getInt32Ty(), Ptr: ElemPtr, IdxList: Cnt);
3688
3689 // Get pointer to location in transfer medium.
3690 // MediumPtr = &medium[warp_id]
3691 Value *MediumPtr = Builder.CreateInBoundsGEP(
3692 Ty: ArrayTy, Ptr: TransferMedium, IdxList: {Builder.getInt64(C: 0), WarpID});
3693 // elem = *elemptr
3694 //*MediumPtr = elem
3695 Value *Elem = Builder.CreateLoad(Ty: CType, Ptr: ElemPtr);
3696 // Store the source element value to the dest element address.
3697 Builder.CreateStore(Val: Elem, Ptr: MediumPtr,
3698 /*IsVolatile*/ isVolatile: true);
3699 Builder.CreateBr(Dest: MergeBB);
3700
3701 // else
3702 emitBlock(BB: ElseBB, CurFn: Builder.GetInsertBlock()->getParent());
3703 Builder.CreateBr(Dest: MergeBB);
3704
3705 // endif
3706 emitBlock(BB: MergeBB, CurFn: Builder.GetInsertBlock()->getParent());
3707 InsertPointOrErrorTy BarrierIP2 =
3708 createBarrier(Loc: LocationDescription(Builder.saveIP(), Loc.DL),
3709 Kind: omp::Directive::OMPD_unknown,
3710 /* ForceSimpleCall */ false,
3711 /* CheckCancelFlag */ true);
3712 if (!BarrierIP2)
3713 return BarrierIP2.takeError();
3714
3715 // Warp 0 copies reduce element from transfer medium
3716 BasicBlock *W0ThenBB = BasicBlock::Create(Context&: Ctx, Name: "then");
3717 BasicBlock *W0ElseBB = BasicBlock::Create(Context&: Ctx, Name: "else");
3718 BasicBlock *W0MergeBB = BasicBlock::Create(Context&: Ctx, Name: "ifcont");
3719
3720 Value *NumWarpsVal =
3721 Builder.CreateLoad(Ty: Builder.getInt32Ty(), Ptr: NumWarpsAddrCast);
3722 // Up to 32 threads in warp 0 are active.
3723 Value *IsActiveThread =
3724 Builder.CreateICmpULT(LHS: GPUThreadID, RHS: NumWarpsVal, Name: "is_active_thread");
3725 Builder.CreateCondBr(Cond: IsActiveThread, True: W0ThenBB, False: W0ElseBB);
3726
3727 emitBlock(BB: W0ThenBB, CurFn: Builder.GetInsertBlock()->getParent());
3728
3729 // SecMediumPtr = &medium[tid]
3730 // SrcMediumVal = *SrcMediumPtr
3731 Value *SrcMediumPtrVal = Builder.CreateInBoundsGEP(
3732 Ty: ArrayTy, Ptr: TransferMedium, IdxList: {Builder.getInt64(C: 0), GPUThreadID});
3733 // TargetElemPtr = (CopyType*)(SrcDataAddr[i]) + I
3734 Value *TargetElemPtrPtr =
3735 Builder.CreateInBoundsGEP(Ty: RedListArrayTy, Ptr: ReduceList,
3736 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0),
3737 ConstantInt::get(Ty: IndexTy, V: En.index())});
3738 Value *TargetElemPtrVal =
3739 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: TargetElemPtrPtr);
3740 Value *TargetElemPtr = TargetElemPtrVal;
3741
3742 if (IsByRefElem && RI.DataPtrPtrGen) {
3743 InsertPointOrErrorTy GenRes =
3744 RI.DataPtrPtrGen(Builder.saveIP(), TargetElemPtr, TargetElemPtr);
3745
3746 if (!GenRes)
3747 return GenRes.takeError();
3748
3749 TargetElemPtr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: TargetElemPtr);
3750 }
3751
3752 if (NumIters > 1)
3753 TargetElemPtr =
3754 Builder.CreateGEP(Ty: Builder.getInt32Ty(), Ptr: TargetElemPtr, IdxList: Cnt);
3755
3756 // *TargetElemPtr = SrcMediumVal;
3757 Value *SrcMediumValue =
3758 Builder.CreateLoad(Ty: CType, Ptr: SrcMediumPtrVal, /*IsVolatile*/ isVolatile: true);
3759 Builder.CreateStore(Val: SrcMediumValue, Ptr: TargetElemPtr);
3760 Builder.CreateBr(Dest: W0MergeBB);
3761
3762 emitBlock(BB: W0ElseBB, CurFn: Builder.GetInsertBlock()->getParent());
3763 Builder.CreateBr(Dest: W0MergeBB);
3764
3765 emitBlock(BB: W0MergeBB, CurFn: Builder.GetInsertBlock()->getParent());
3766
3767 if (NumIters > 1) {
3768 Cnt = Builder.CreateNSWAdd(
3769 LHS: Cnt, RHS: ConstantInt::get(Ty: Builder.getInt32Ty(), /*V=*/1));
3770 Builder.CreateStore(Val: Cnt, Ptr: CntAddr, /*Volatile=*/isVolatile: false);
3771
3772 auto *CurFn = Builder.GetInsertBlock()->getParent();
3773 emitBranch(Target: PrecondBB);
3774 emitBlock(BB: ExitBB, CurFn);
3775 }
3776 RealTySize %= TySize;
3777 }
3778 }
3779
3780 Builder.CreateRetVoid();
3781 Builder.restoreIP(IP: SavedIP);
3782
3783 return WcFunc;
3784}
3785
3786Expected<Function *> OpenMPIRBuilder::emitShuffleAndReduceFunction(
3787 ArrayRef<ReductionInfo> ReductionInfos, Function *ReduceFn,
3788 AttributeList FuncAttrs, ArrayRef<bool> IsByRef) {
3789 LLVMContext &Ctx = M.getContext();
3790 FunctionType *FuncTy =
3791 FunctionType::get(Result: Builder.getVoidTy(),
3792 Params: {Builder.getPtrTy(), Builder.getInt16Ty(),
3793 Builder.getInt16Ty(), Builder.getInt16Ty()},
3794 /* IsVarArg */ isVarArg: false);
3795 Function *SarFunc =
3796 Function::Create(Ty: FuncTy, Linkage: GlobalVariable::InternalLinkage,
3797 N: "_omp_reduction_shuffle_and_reduce_func", M: &M);
3798 SarFunc->setCallingConv(Config.getRuntimeCC());
3799 SarFunc->setAttributes(FuncAttrs);
3800 SarFunc->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
3801 SarFunc->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
3802 SarFunc->addParamAttr(ArgNo: 2, Kind: Attribute::NoUndef);
3803 SarFunc->addParamAttr(ArgNo: 3, Kind: Attribute::NoUndef);
3804 SarFunc->addParamAttr(ArgNo: 1, Kind: Attribute::SExt);
3805 SarFunc->addParamAttr(ArgNo: 2, Kind: Attribute::SExt);
3806 SarFunc->addParamAttr(ArgNo: 3, Kind: Attribute::SExt);
3807 BasicBlock *EntryBB = BasicBlock::Create(Context&: M.getContext(), Name: "entry", Parent: SarFunc);
3808 Builder.SetInsertPoint(EntryBB);
3809
3810 // Thread local Reduce list used to host the values of data to be reduced.
3811 Argument *ReduceListArg = SarFunc->getArg(i: 0);
3812 // Current lane id; could be logical.
3813 Argument *LaneIDArg = SarFunc->getArg(i: 1);
3814 // Offset of the remote source lane relative to the current lane.
3815 Argument *RemoteLaneOffsetArg = SarFunc->getArg(i: 2);
3816 // Algorithm version. This is expected to be known at compile time.
3817 Argument *AlgoVerArg = SarFunc->getArg(i: 3);
3818
3819 Type *ReduceListArgType = ReduceListArg->getType();
3820 Type *LaneIDArgType = LaneIDArg->getType();
3821 Type *LaneIDArgPtrType = Builder.getPtrTy(AddrSpace: 0);
3822 Value *ReduceListAlloca = Builder.CreateAlloca(
3823 Ty: ReduceListArgType, ArraySize: nullptr, Name: ReduceListArg->getName() + ".addr");
3824 Value *LaneIdAlloca = Builder.CreateAlloca(Ty: LaneIDArgType, ArraySize: nullptr,
3825 Name: LaneIDArg->getName() + ".addr");
3826 Value *RemoteLaneOffsetAlloca = Builder.CreateAlloca(
3827 Ty: LaneIDArgType, ArraySize: nullptr, Name: RemoteLaneOffsetArg->getName() + ".addr");
3828 Value *AlgoVerAlloca = Builder.CreateAlloca(Ty: LaneIDArgType, ArraySize: nullptr,
3829 Name: AlgoVerArg->getName() + ".addr");
3830 ArrayType *RedListArrayTy =
3831 ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: ReductionInfos.size());
3832
3833 // Create a local thread-private variable to host the Reduce list
3834 // from a remote lane.
3835 Instruction *RemoteReductionListAlloca = Builder.CreateAlloca(
3836 Ty: RedListArrayTy, ArraySize: nullptr, Name: ".omp.reduction.remote_reduce_list");
3837
3838 Value *ReduceListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
3839 V: ReduceListAlloca, DestTy: ReduceListArgType,
3840 Name: ReduceListAlloca->getName() + ".ascast");
3841 Value *LaneIdAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
3842 V: LaneIdAlloca, DestTy: LaneIDArgPtrType, Name: LaneIdAlloca->getName() + ".ascast");
3843 Value *RemoteLaneOffsetAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
3844 V: RemoteLaneOffsetAlloca, DestTy: LaneIDArgPtrType,
3845 Name: RemoteLaneOffsetAlloca->getName() + ".ascast");
3846 Value *AlgoVerAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
3847 V: AlgoVerAlloca, DestTy: LaneIDArgPtrType, Name: AlgoVerAlloca->getName() + ".ascast");
3848 Value *RemoteListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
3849 V: RemoteReductionListAlloca, DestTy: Builder.getPtrTy(),
3850 Name: RemoteReductionListAlloca->getName() + ".ascast");
3851
3852 Builder.CreateStore(Val: ReduceListArg, Ptr: ReduceListAddrCast);
3853 Builder.CreateStore(Val: LaneIDArg, Ptr: LaneIdAddrCast);
3854 Builder.CreateStore(Val: RemoteLaneOffsetArg, Ptr: RemoteLaneOffsetAddrCast);
3855 Builder.CreateStore(Val: AlgoVerArg, Ptr: AlgoVerAddrCast);
3856
3857 Value *ReduceList = Builder.CreateLoad(Ty: ReduceListArgType, Ptr: ReduceListAddrCast);
3858 Value *LaneId = Builder.CreateLoad(Ty: LaneIDArgType, Ptr: LaneIdAddrCast);
3859 Value *RemoteLaneOffset =
3860 Builder.CreateLoad(Ty: LaneIDArgType, Ptr: RemoteLaneOffsetAddrCast);
3861 Value *AlgoVer = Builder.CreateLoad(Ty: LaneIDArgType, Ptr: AlgoVerAddrCast);
3862
3863 InsertPointTy AllocaIP = getInsertPointAfterInstr(I: RemoteReductionListAlloca);
3864
3865 // This loop iterates through the list of reduce elements and copies,
3866 // element by element, from a remote lane in the warp to RemoteReduceList,
3867 // hosted on the thread's stack.
3868 Error EmitRedLsCpRes = emitReductionListCopy(
3869 AllocaIP, Action: CopyAction::RemoteLaneToThread, ReductionArrayTy: RedListArrayTy, ReductionInfos,
3870 SrcBase: ReduceList, DestBase: RemoteListAddrCast, IsByRef,
3871 CopyOptions: {.RemoteLaneOffset: RemoteLaneOffset, .ScratchpadIndex: nullptr, .ScratchpadWidth: nullptr});
3872
3873 if (EmitRedLsCpRes)
3874 return EmitRedLsCpRes;
3875
3876 // The actions to be performed on the Remote Reduce list is dependent
3877 // on the algorithm version.
3878 //
3879 // if (AlgoVer==0) || (AlgoVer==1 && (LaneId < Offset)) || (AlgoVer==2 &&
3880 // LaneId % 2 == 0 && Offset > 0):
3881 // do the reduction value aggregation
3882 //
3883 // The thread local variable Reduce list is mutated in place to host the
3884 // reduced data, which is the aggregated value produced from local and
3885 // remote lanes.
3886 //
3887 // Note that AlgoVer is expected to be a constant integer known at compile
3888 // time.
3889 // When AlgoVer==0, the first conjunction evaluates to true, making
3890 // the entire predicate true during compile time.
3891 // When AlgoVer==1, the second conjunction has only the second part to be
3892 // evaluated during runtime. Other conjunctions evaluates to false
3893 // during compile time.
3894 // When AlgoVer==2, the third conjunction has only the second part to be
3895 // evaluated during runtime. Other conjunctions evaluates to false
3896 // during compile time.
3897 Value *CondAlgo0 = Builder.CreateIsNull(Arg: AlgoVer);
3898 Value *Algo1 = Builder.CreateICmpEQ(LHS: AlgoVer, RHS: Builder.getInt16(C: 1));
3899 Value *LaneComp = Builder.CreateICmpULT(LHS: LaneId, RHS: RemoteLaneOffset);
3900 Value *CondAlgo1 = Builder.CreateAnd(LHS: Algo1, RHS: LaneComp);
3901 Value *Algo2 = Builder.CreateICmpEQ(LHS: AlgoVer, RHS: Builder.getInt16(C: 2));
3902 Value *LaneIdAnd1 = Builder.CreateAnd(LHS: LaneId, RHS: Builder.getInt16(C: 1));
3903 Value *LaneIdComp = Builder.CreateIsNull(Arg: LaneIdAnd1);
3904 Value *Algo2AndLaneIdComp = Builder.CreateAnd(LHS: Algo2, RHS: LaneIdComp);
3905 Value *RemoteOffsetComp =
3906 Builder.CreateICmpSGT(LHS: RemoteLaneOffset, RHS: Builder.getInt16(C: 0));
3907 Value *CondAlgo2 = Builder.CreateAnd(LHS: Algo2AndLaneIdComp, RHS: RemoteOffsetComp);
3908 Value *CA0OrCA1 = Builder.CreateOr(LHS: CondAlgo0, RHS: CondAlgo1);
3909 Value *CondReduce = Builder.CreateOr(LHS: CA0OrCA1, RHS: CondAlgo2);
3910
3911 BasicBlock *ThenBB = BasicBlock::Create(Context&: Ctx, Name: "then");
3912 BasicBlock *ElseBB = BasicBlock::Create(Context&: Ctx, Name: "else");
3913 BasicBlock *MergeBB = BasicBlock::Create(Context&: Ctx, Name: "ifcont");
3914
3915 Builder.CreateCondBr(Cond: CondReduce, True: ThenBB, False: ElseBB);
3916 emitBlock(BB: ThenBB, CurFn: Builder.GetInsertBlock()->getParent());
3917 Value *LocalReduceListPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
3918 V: ReduceList, DestTy: Builder.getPtrTy());
3919 Value *RemoteReduceListPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
3920 V: RemoteListAddrCast, DestTy: Builder.getPtrTy());
3921 createRuntimeFunctionCall(Callee: ReduceFn, Args: {LocalReduceListPtr, RemoteReduceListPtr})
3922 ->addFnAttr(Kind: Attribute::NoUnwind);
3923 Builder.CreateBr(Dest: MergeBB);
3924
3925 emitBlock(BB: ElseBB, CurFn: Builder.GetInsertBlock()->getParent());
3926 Builder.CreateBr(Dest: MergeBB);
3927
3928 emitBlock(BB: MergeBB, CurFn: Builder.GetInsertBlock()->getParent());
3929
3930 // if (AlgoVer==1 && (LaneId >= Offset)) copy Remote Reduce list to local
3931 // Reduce list.
3932 Algo1 = Builder.CreateICmpEQ(LHS: AlgoVer, RHS: Builder.getInt16(C: 1));
3933 Value *LaneIdGtOffset = Builder.CreateICmpUGE(LHS: LaneId, RHS: RemoteLaneOffset);
3934 Value *CondCopy = Builder.CreateAnd(LHS: Algo1, RHS: LaneIdGtOffset);
3935
3936 BasicBlock *CpyThenBB = BasicBlock::Create(Context&: Ctx, Name: "then");
3937 BasicBlock *CpyElseBB = BasicBlock::Create(Context&: Ctx, Name: "else");
3938 BasicBlock *CpyMergeBB = BasicBlock::Create(Context&: Ctx, Name: "ifcont");
3939 Builder.CreateCondBr(Cond: CondCopy, True: CpyThenBB, False: CpyElseBB);
3940
3941 emitBlock(BB: CpyThenBB, CurFn: Builder.GetInsertBlock()->getParent());
3942
3943 EmitRedLsCpRes = emitReductionListCopy(
3944 AllocaIP, Action: CopyAction::ThreadCopy, ReductionArrayTy: RedListArrayTy, ReductionInfos,
3945 SrcBase: RemoteListAddrCast, DestBase: ReduceList, IsByRef);
3946
3947 if (EmitRedLsCpRes)
3948 return EmitRedLsCpRes;
3949
3950 Builder.CreateBr(Dest: CpyMergeBB);
3951
3952 emitBlock(BB: CpyElseBB, CurFn: Builder.GetInsertBlock()->getParent());
3953 Builder.CreateBr(Dest: CpyMergeBB);
3954
3955 emitBlock(BB: CpyMergeBB, CurFn: Builder.GetInsertBlock()->getParent());
3956
3957 Builder.CreateRetVoid();
3958
3959 return SarFunc;
3960}
3961
3962OpenMPIRBuilder::InsertPointOrErrorTy
3963OpenMPIRBuilder::generateReductionDescriptor(
3964 Value *DescriptorAddr, Value *DataPtr, Value *SrcDescriptorAddr,
3965 Type *DescriptorType,
3966 function_ref<InsertPointOrErrorTy(InsertPointTy, Value *, Value *&)>
3967 DataPtrPtrGen) {
3968
3969 // Copy the source descriptor to preserve all metadata (rank, extents,
3970 // strides, etc.)
3971 Value *DescriptorSize =
3972 Builder.getInt64(C: M.getDataLayout().getTypeStoreSize(Ty: DescriptorType));
3973 Builder.CreateMemCpy(
3974 Dst: DescriptorAddr, DstAlign: M.getDataLayout().getPrefTypeAlign(Ty: DescriptorType),
3975 Src: SrcDescriptorAddr, SrcAlign: M.getDataLayout().getPrefTypeAlign(Ty: DescriptorType),
3976 Size: DescriptorSize);
3977
3978 // Update the base pointer field to point to the local shuffled data
3979 Value *DataPtrField;
3980 InsertPointOrErrorTy GenResult =
3981 DataPtrPtrGen(Builder.saveIP(), DescriptorAddr, DataPtrField);
3982
3983 if (!GenResult)
3984 return GenResult.takeError();
3985
3986 Builder.CreateStore(Val: Builder.CreatePointerBitCastOrAddrSpaceCast(
3987 V: DataPtr, DestTy: Builder.getPtrTy(), Name: ".ascast"),
3988 Ptr: DataPtrField);
3989
3990 return Builder.saveIP();
3991}
3992
3993Expected<Value *> OpenMPIRBuilder::createReductionDescriptorCopy(
3994 InsertPointTy AllocaIP, const ReductionInfo &RI, Value *DataPtr,
3995 Value *SrcDescriptorAddr, Type *DescriptorPtrTy, const Twine &Name) {
3996 InsertPointTy OldIP = Builder.saveIP();
3997 Builder.restoreIP(IP: AllocaIP);
3998
3999 AllocaInst *DescriptorAlloca =
4000 Builder.CreateAlloca(Ty: RI.ByRefAllocatedType, ArraySize: nullptr, Name);
4001 DescriptorAlloca->setAlignment(
4002 M.getDataLayout().getPrefTypeAlign(Ty: RI.ByRefAllocatedType));
4003 Value *DescriptorAddr = Builder.CreatePointerBitCastOrAddrSpaceCast(
4004 V: DescriptorAlloca, DestTy: DescriptorPtrTy,
4005 Name: DescriptorAlloca->getName() + ".ascast");
4006
4007 Builder.restoreIP(IP: OldIP);
4008
4009 InsertPointOrErrorTy GenResult =
4010 generateReductionDescriptor(DescriptorAddr, DataPtr, SrcDescriptorAddr,
4011 DescriptorType: RI.ByRefAllocatedType, DataPtrPtrGen: RI.DataPtrPtrGen);
4012 if (!GenResult)
4013 return GenResult.takeError();
4014
4015 return DescriptorAddr;
4016}
4017
4018Expected<Function *> OpenMPIRBuilder::emitListToGlobalCopyFunction(
4019 ArrayRef<ReductionInfo> ReductionInfos, Type *ReductionsBufferTy,
4020 AttributeList FuncAttrs, ArrayRef<bool> IsByRef) {
4021 OpenMPIRBuilder::InsertPointTy OldIP = Builder.saveIP();
4022 LLVMContext &Ctx = M.getContext();
4023 FunctionType *FuncTy = FunctionType::get(
4024 Result: Builder.getVoidTy(),
4025 Params: {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},
4026 /* IsVarArg */ isVarArg: false);
4027 Function *LtGCFunc =
4028 Function::Create(Ty: FuncTy, Linkage: GlobalVariable::InternalLinkage,
4029 N: "_omp_reduction_list_to_global_copy_func", M: &M);
4030 LtGCFunc->setAttributes(FuncAttrs);
4031 LtGCFunc->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
4032 LtGCFunc->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
4033 LtGCFunc->addParamAttr(ArgNo: 2, Kind: Attribute::NoUndef);
4034
4035 BasicBlock *EntryBlock = BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: LtGCFunc);
4036 Builder.SetInsertPoint(EntryBlock);
4037
4038 // Buffer: global reduction buffer.
4039 Argument *BufferArg = LtGCFunc->getArg(i: 0);
4040 // Idx: index of the buffer.
4041 Argument *IdxArg = LtGCFunc->getArg(i: 1);
4042 // ReduceList: thread local Reduce list.
4043 Argument *ReduceListArg = LtGCFunc->getArg(i: 2);
4044
4045 Value *BufferArgAlloca = Builder.CreateAlloca(Ty: Builder.getPtrTy(), ArraySize: nullptr,
4046 Name: BufferArg->getName() + ".addr");
4047 Value *IdxArgAlloca = Builder.CreateAlloca(Ty: Builder.getInt32Ty(), ArraySize: nullptr,
4048 Name: IdxArg->getName() + ".addr");
4049 Value *ReduceListArgAlloca = Builder.CreateAlloca(
4050 Ty: Builder.getPtrTy(), ArraySize: nullptr, Name: ReduceListArg->getName() + ".addr");
4051 Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4052 V: BufferArgAlloca, DestTy: Builder.getPtrTy(),
4053 Name: BufferArgAlloca->getName() + ".ascast");
4054 Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4055 V: IdxArgAlloca, DestTy: Builder.getPtrTy(), Name: IdxArgAlloca->getName() + ".ascast");
4056 Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4057 V: ReduceListArgAlloca, DestTy: Builder.getPtrTy(),
4058 Name: ReduceListArgAlloca->getName() + ".ascast");
4059
4060 Builder.CreateStore(Val: BufferArg, Ptr: BufferArgAddrCast);
4061 Builder.CreateStore(Val: IdxArg, Ptr: IdxArgAddrCast);
4062 Builder.CreateStore(Val: ReduceListArg, Ptr: ReduceListArgAddrCast);
4063
4064 Value *LocalReduceList =
4065 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ReduceListArgAddrCast);
4066 Value *BufferArgVal =
4067 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: BufferArgAddrCast);
4068 Value *Idxs[] = {Builder.CreateLoad(Ty: Builder.getInt32Ty(), Ptr: IdxArgAddrCast)};
4069 Type *IndexTy = Builder.getIndexTy(
4070 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
4071 for (auto En : enumerate(First&: ReductionInfos)) {
4072 const ReductionInfo &RI = En.value();
4073 auto *RedListArrayTy =
4074 ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: ReductionInfos.size());
4075 // Reduce element = LocalReduceList[i]
4076 Value *ElemPtrPtr = Builder.CreateInBoundsGEP(
4077 Ty: RedListArrayTy, Ptr: LocalReduceList,
4078 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
4079 // elemptr = ((CopyType*)(elemptrptr)) + I
4080 Value *ElemPtr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ElemPtrPtr);
4081
4082 // Global = Buffer.VD[Idx];
4083 Value *BufferVD =
4084 Builder.CreateInBoundsGEP(Ty: ReductionsBufferTy, Ptr: BufferArgVal, IdxList: Idxs);
4085 Value *GlobVal = Builder.CreateConstInBoundsGEP2_32(
4086 Ty: ReductionsBufferTy, Ptr: BufferVD, Idx0: 0, Idx1: En.index());
4087
4088 switch (RI.EvaluationKind) {
4089 case EvalKind::Scalar: {
4090 Value *TargetElement;
4091
4092 if (IsByRef.empty() || !IsByRef[En.index()]) {
4093 TargetElement = Builder.CreateLoad(Ty: RI.ElementType, Ptr: ElemPtr);
4094 } else {
4095 if (RI.DataPtrPtrGen) {
4096 InsertPointOrErrorTy GenResult =
4097 RI.DataPtrPtrGen(Builder.saveIP(), ElemPtr, ElemPtr);
4098
4099 if (!GenResult)
4100 return GenResult.takeError();
4101
4102 ElemPtr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ElemPtr);
4103 }
4104 TargetElement = Builder.CreateLoad(Ty: RI.ByRefElementType, Ptr: ElemPtr);
4105 }
4106
4107 Builder.CreateStore(Val: TargetElement, Ptr: GlobVal);
4108 break;
4109 }
4110 case EvalKind::Complex: {
4111 Value *SrcRealPtr = Builder.CreateConstInBoundsGEP2_32(
4112 Ty: RI.ElementType, Ptr: ElemPtr, Idx0: 0, Idx1: 0, Name: ".realp");
4113 Value *SrcReal = Builder.CreateLoad(
4114 Ty: RI.ElementType->getStructElementType(N: 0), Ptr: SrcRealPtr, Name: ".real");
4115 Value *SrcImgPtr = Builder.CreateConstInBoundsGEP2_32(
4116 Ty: RI.ElementType, Ptr: ElemPtr, Idx0: 0, Idx1: 1, Name: ".imagp");
4117 Value *SrcImg = Builder.CreateLoad(
4118 Ty: RI.ElementType->getStructElementType(N: 1), Ptr: SrcImgPtr, Name: ".imag");
4119
4120 Value *DestRealPtr = Builder.CreateConstInBoundsGEP2_32(
4121 Ty: RI.ElementType, Ptr: GlobVal, Idx0: 0, Idx1: 0, Name: ".realp");
4122 Value *DestImgPtr = Builder.CreateConstInBoundsGEP2_32(
4123 Ty: RI.ElementType, Ptr: GlobVal, Idx0: 0, Idx1: 1, Name: ".imagp");
4124 Builder.CreateStore(Val: SrcReal, Ptr: DestRealPtr);
4125 Builder.CreateStore(Val: SrcImg, Ptr: DestImgPtr);
4126 break;
4127 }
4128 case EvalKind::Aggregate: {
4129 Value *SizeVal =
4130 Builder.getInt64(C: M.getDataLayout().getTypeStoreSize(Ty: RI.ElementType));
4131 Builder.CreateMemCpy(
4132 Dst: GlobVal, DstAlign: M.getDataLayout().getPrefTypeAlign(Ty: RI.ElementType), Src: ElemPtr,
4133 SrcAlign: M.getDataLayout().getPrefTypeAlign(Ty: RI.ElementType), Size: SizeVal, isVolatile: false);
4134 break;
4135 }
4136 }
4137 }
4138
4139 Builder.CreateRetVoid();
4140 Builder.restoreIP(IP: OldIP);
4141 return LtGCFunc;
4142}
4143
4144Expected<Function *> OpenMPIRBuilder::emitListToGlobalReduceFunction(
4145 ArrayRef<ReductionInfo> ReductionInfos, Function *ReduceFn,
4146 Type *ReductionsBufferTy, AttributeList FuncAttrs, ArrayRef<bool> IsByRef) {
4147 OpenMPIRBuilder::InsertPointTy OldIP = Builder.saveIP();
4148 LLVMContext &Ctx = M.getContext();
4149 FunctionType *FuncTy = FunctionType::get(
4150 Result: Builder.getVoidTy(),
4151 Params: {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},
4152 /* IsVarArg */ isVarArg: false);
4153 Function *LtGRFunc =
4154 Function::Create(Ty: FuncTy, Linkage: GlobalVariable::InternalLinkage,
4155 N: "_omp_reduction_list_to_global_reduce_func", M: &M);
4156 LtGRFunc->setAttributes(FuncAttrs);
4157 LtGRFunc->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
4158 LtGRFunc->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
4159 LtGRFunc->addParamAttr(ArgNo: 2, Kind: Attribute::NoUndef);
4160
4161 BasicBlock *EntryBlock = BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: LtGRFunc);
4162 Builder.SetInsertPoint(EntryBlock);
4163
4164 // Buffer: global reduction buffer.
4165 Argument *BufferArg = LtGRFunc->getArg(i: 0);
4166 // Idx: index of the buffer.
4167 Argument *IdxArg = LtGRFunc->getArg(i: 1);
4168 // ReduceList: thread local Reduce list.
4169 Argument *ReduceListArg = LtGRFunc->getArg(i: 2);
4170
4171 Value *BufferArgAlloca = Builder.CreateAlloca(Ty: Builder.getPtrTy(), ArraySize: nullptr,
4172 Name: BufferArg->getName() + ".addr");
4173 Value *IdxArgAlloca = Builder.CreateAlloca(Ty: Builder.getInt32Ty(), ArraySize: nullptr,
4174 Name: IdxArg->getName() + ".addr");
4175 Value *ReduceListArgAlloca = Builder.CreateAlloca(
4176 Ty: Builder.getPtrTy(), ArraySize: nullptr, Name: ReduceListArg->getName() + ".addr");
4177 auto *RedListArrayTy =
4178 ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: ReductionInfos.size());
4179
4180 // 1. Build a list of reduction variables.
4181 // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
4182 Value *LocalReduceList =
4183 Builder.CreateAlloca(Ty: RedListArrayTy, ArraySize: nullptr, Name: ".omp.reduction.red_list");
4184
4185 InsertPointTy AllocaIP{EntryBlock, EntryBlock->begin()};
4186
4187 Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4188 V: BufferArgAlloca, DestTy: Builder.getPtrTy(),
4189 Name: BufferArgAlloca->getName() + ".ascast");
4190 Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4191 V: IdxArgAlloca, DestTy: Builder.getPtrTy(), Name: IdxArgAlloca->getName() + ".ascast");
4192 Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4193 V: ReduceListArgAlloca, DestTy: Builder.getPtrTy(),
4194 Name: ReduceListArgAlloca->getName() + ".ascast");
4195 Value *LocalReduceListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4196 V: LocalReduceList, DestTy: Builder.getPtrTy(),
4197 Name: LocalReduceList->getName() + ".ascast");
4198
4199 Builder.CreateStore(Val: BufferArg, Ptr: BufferArgAddrCast);
4200 Builder.CreateStore(Val: IdxArg, Ptr: IdxArgAddrCast);
4201 Builder.CreateStore(Val: ReduceListArg, Ptr: ReduceListArgAddrCast);
4202
4203 Value *BufferVal = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: BufferArgAddrCast);
4204 Value *Idxs[] = {Builder.CreateLoad(Ty: Builder.getInt32Ty(), Ptr: IdxArgAddrCast)};
4205 Type *IndexTy = Builder.getIndexTy(
4206 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
4207 for (auto En : enumerate(First&: ReductionInfos)) {
4208 const ReductionInfo &RI = En.value();
4209
4210 Value *TargetElementPtrPtr = Builder.CreateInBoundsGEP(
4211 Ty: RedListArrayTy, Ptr: LocalReduceListAddrCast,
4212 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
4213 Value *BufferVD =
4214 Builder.CreateInBoundsGEP(Ty: ReductionsBufferTy, Ptr: BufferVal, IdxList: Idxs);
4215 // Global = Buffer.VD[Idx];
4216 Value *GlobValPtr = Builder.CreateConstInBoundsGEP2_32(
4217 Ty: ReductionsBufferTy, Ptr: BufferVD, Idx0: 0, Idx1: En.index());
4218
4219 if (!IsByRef.empty() && IsByRef[En.index()] && RI.DataPtrPtrGen) {
4220 // Get source descriptor from the reduce list argument
4221 Value *ReduceList =
4222 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ReduceListArgAddrCast);
4223 Value *SrcElementPtrPtr =
4224 Builder.CreateInBoundsGEP(Ty: RedListArrayTy, Ptr: ReduceList,
4225 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0),
4226 ConstantInt::get(Ty: IndexTy, V: En.index())});
4227 Value *SrcDescriptorAddr =
4228 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: SrcElementPtrPtr);
4229
4230 // Copy descriptor from source and update base_ptr to global buffer data
4231 Expected<Value *> ByRefAlloc = createReductionDescriptorCopy(
4232 AllocaIP, RI, DataPtr: GlobValPtr, SrcDescriptorAddr, DescriptorPtrTy: Builder.getPtrTy());
4233 if (!ByRefAlloc)
4234 return ByRefAlloc.takeError();
4235
4236 Builder.CreateStore(Val: *ByRefAlloc, Ptr: TargetElementPtrPtr);
4237 } else {
4238 Builder.CreateStore(Val: GlobValPtr, Ptr: TargetElementPtrPtr);
4239 }
4240 }
4241
4242 // Call reduce_function(GlobalReduceList, ReduceList)
4243 Value *ReduceList =
4244 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ReduceListArgAddrCast);
4245 createRuntimeFunctionCall(Callee: ReduceFn, Args: {LocalReduceListAddrCast, ReduceList})
4246 ->addFnAttr(Kind: Attribute::NoUnwind);
4247 Builder.CreateRetVoid();
4248 Builder.restoreIP(IP: OldIP);
4249 return LtGRFunc;
4250}
4251
4252Expected<Function *> OpenMPIRBuilder::emitGlobalToListCopyFunction(
4253 ArrayRef<ReductionInfo> ReductionInfos, Type *ReductionsBufferTy,
4254 AttributeList FuncAttrs, ArrayRef<bool> IsByRef) {
4255 OpenMPIRBuilder::InsertPointTy OldIP = Builder.saveIP();
4256 LLVMContext &Ctx = M.getContext();
4257 FunctionType *FuncTy = FunctionType::get(
4258 Result: Builder.getVoidTy(),
4259 Params: {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},
4260 /* IsVarArg */ isVarArg: false);
4261 Function *GtLCFunc =
4262 Function::Create(Ty: FuncTy, Linkage: GlobalVariable::InternalLinkage,
4263 N: "_omp_reduction_global_to_list_copy_func", M: &M);
4264 GtLCFunc->setAttributes(FuncAttrs);
4265 GtLCFunc->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
4266 GtLCFunc->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
4267 GtLCFunc->addParamAttr(ArgNo: 2, Kind: Attribute::NoUndef);
4268
4269 BasicBlock *EntryBlock = BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: GtLCFunc);
4270 Builder.SetInsertPoint(EntryBlock);
4271
4272 // Buffer: global reduction buffer.
4273 Argument *BufferArg = GtLCFunc->getArg(i: 0);
4274 // Idx: index of the buffer.
4275 Argument *IdxArg = GtLCFunc->getArg(i: 1);
4276 // ReduceList: thread local Reduce list.
4277 Argument *ReduceListArg = GtLCFunc->getArg(i: 2);
4278
4279 Value *BufferArgAlloca = Builder.CreateAlloca(Ty: Builder.getPtrTy(), ArraySize: nullptr,
4280 Name: BufferArg->getName() + ".addr");
4281 Value *IdxArgAlloca = Builder.CreateAlloca(Ty: Builder.getInt32Ty(), ArraySize: nullptr,
4282 Name: IdxArg->getName() + ".addr");
4283 Value *ReduceListArgAlloca = Builder.CreateAlloca(
4284 Ty: Builder.getPtrTy(), ArraySize: nullptr, Name: ReduceListArg->getName() + ".addr");
4285 Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4286 V: BufferArgAlloca, DestTy: Builder.getPtrTy(),
4287 Name: BufferArgAlloca->getName() + ".ascast");
4288 Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4289 V: IdxArgAlloca, DestTy: Builder.getPtrTy(), Name: IdxArgAlloca->getName() + ".ascast");
4290 Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4291 V: ReduceListArgAlloca, DestTy: Builder.getPtrTy(),
4292 Name: ReduceListArgAlloca->getName() + ".ascast");
4293 Builder.CreateStore(Val: BufferArg, Ptr: BufferArgAddrCast);
4294 Builder.CreateStore(Val: IdxArg, Ptr: IdxArgAddrCast);
4295 Builder.CreateStore(Val: ReduceListArg, Ptr: ReduceListArgAddrCast);
4296
4297 Value *LocalReduceList =
4298 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ReduceListArgAddrCast);
4299 Value *BufferVal = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: BufferArgAddrCast);
4300 Value *Idxs[] = {Builder.CreateLoad(Ty: Builder.getInt32Ty(), Ptr: IdxArgAddrCast)};
4301 Type *IndexTy = Builder.getIndexTy(
4302 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
4303 for (auto En : enumerate(First&: ReductionInfos)) {
4304 const OpenMPIRBuilder::ReductionInfo &RI = En.value();
4305 auto *RedListArrayTy =
4306 ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: ReductionInfos.size());
4307 // Reduce element = LocalReduceList[i]
4308 Value *ElemPtrPtr = Builder.CreateInBoundsGEP(
4309 Ty: RedListArrayTy, Ptr: LocalReduceList,
4310 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
4311 // elemptr = ((CopyType*)(elemptrptr)) + I
4312 Value *ElemPtr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ElemPtrPtr);
4313 // Global = Buffer.VD[Idx];
4314 Value *BufferVD =
4315 Builder.CreateInBoundsGEP(Ty: ReductionsBufferTy, Ptr: BufferVal, IdxList: Idxs);
4316 Value *GlobValPtr = Builder.CreateConstInBoundsGEP2_32(
4317 Ty: ReductionsBufferTy, Ptr: BufferVD, Idx0: 0, Idx1: En.index());
4318
4319 switch (RI.EvaluationKind) {
4320 case EvalKind::Scalar: {
4321 Type *ElemType = RI.ElementType;
4322
4323 if (!IsByRef.empty() && IsByRef[En.index()]) {
4324 ElemType = RI.ByRefElementType;
4325 if (RI.DataPtrPtrGen) {
4326 InsertPointOrErrorTy GenResult =
4327 RI.DataPtrPtrGen(Builder.saveIP(), ElemPtr, ElemPtr);
4328
4329 if (!GenResult)
4330 return GenResult.takeError();
4331
4332 ElemPtr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ElemPtr);
4333 }
4334 }
4335
4336 Value *TargetElement = Builder.CreateLoad(Ty: ElemType, Ptr: GlobValPtr);
4337 Builder.CreateStore(Val: TargetElement, Ptr: ElemPtr);
4338 break;
4339 }
4340 case EvalKind::Complex: {
4341 Value *SrcRealPtr = Builder.CreateConstInBoundsGEP2_32(
4342 Ty: RI.ElementType, Ptr: GlobValPtr, Idx0: 0, Idx1: 0, Name: ".realp");
4343 Value *SrcReal = Builder.CreateLoad(
4344 Ty: RI.ElementType->getStructElementType(N: 0), Ptr: SrcRealPtr, Name: ".real");
4345 Value *SrcImgPtr = Builder.CreateConstInBoundsGEP2_32(
4346 Ty: RI.ElementType, Ptr: GlobValPtr, Idx0: 0, Idx1: 1, Name: ".imagp");
4347 Value *SrcImg = Builder.CreateLoad(
4348 Ty: RI.ElementType->getStructElementType(N: 1), Ptr: SrcImgPtr, Name: ".imag");
4349
4350 Value *DestRealPtr = Builder.CreateConstInBoundsGEP2_32(
4351 Ty: RI.ElementType, Ptr: ElemPtr, Idx0: 0, Idx1: 0, Name: ".realp");
4352 Value *DestImgPtr = Builder.CreateConstInBoundsGEP2_32(
4353 Ty: RI.ElementType, Ptr: ElemPtr, Idx0: 0, Idx1: 1, Name: ".imagp");
4354 Builder.CreateStore(Val: SrcReal, Ptr: DestRealPtr);
4355 Builder.CreateStore(Val: SrcImg, Ptr: DestImgPtr);
4356 break;
4357 }
4358 case EvalKind::Aggregate: {
4359 Value *SizeVal =
4360 Builder.getInt64(C: M.getDataLayout().getTypeStoreSize(Ty: RI.ElementType));
4361 Builder.CreateMemCpy(
4362 Dst: ElemPtr, DstAlign: M.getDataLayout().getPrefTypeAlign(Ty: RI.ElementType),
4363 Src: GlobValPtr, SrcAlign: M.getDataLayout().getPrefTypeAlign(Ty: RI.ElementType),
4364 Size: SizeVal, isVolatile: false);
4365 break;
4366 }
4367 }
4368 }
4369
4370 Builder.CreateRetVoid();
4371 Builder.restoreIP(IP: OldIP);
4372 return GtLCFunc;
4373}
4374
4375Expected<Function *> OpenMPIRBuilder::emitGlobalToListReduceFunction(
4376 ArrayRef<ReductionInfo> ReductionInfos, Function *ReduceFn,
4377 Type *ReductionsBufferTy, AttributeList FuncAttrs, ArrayRef<bool> IsByRef) {
4378 OpenMPIRBuilder::InsertPointTy OldIP = Builder.saveIP();
4379 LLVMContext &Ctx = M.getContext();
4380 auto *FuncTy = FunctionType::get(
4381 Result: Builder.getVoidTy(),
4382 Params: {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},
4383 /* IsVarArg */ isVarArg: false);
4384 Function *GtLRFunc =
4385 Function::Create(Ty: FuncTy, Linkage: GlobalVariable::InternalLinkage,
4386 N: "_omp_reduction_global_to_list_reduce_func", M: &M);
4387 GtLRFunc->setAttributes(FuncAttrs);
4388 GtLRFunc->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
4389 GtLRFunc->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
4390 GtLRFunc->addParamAttr(ArgNo: 2, Kind: Attribute::NoUndef);
4391
4392 BasicBlock *EntryBlock = BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: GtLRFunc);
4393 Builder.SetInsertPoint(EntryBlock);
4394
4395 // Buffer: global reduction buffer.
4396 Argument *BufferArg = GtLRFunc->getArg(i: 0);
4397 // Idx: index of the buffer.
4398 Argument *IdxArg = GtLRFunc->getArg(i: 1);
4399 // ReduceList: thread local Reduce list.
4400 Argument *ReduceListArg = GtLRFunc->getArg(i: 2);
4401
4402 Value *BufferArgAlloca = Builder.CreateAlloca(Ty: Builder.getPtrTy(), ArraySize: nullptr,
4403 Name: BufferArg->getName() + ".addr");
4404 Value *IdxArgAlloca = Builder.CreateAlloca(Ty: Builder.getInt32Ty(), ArraySize: nullptr,
4405 Name: IdxArg->getName() + ".addr");
4406 Value *ReduceListArgAlloca = Builder.CreateAlloca(
4407 Ty: Builder.getPtrTy(), ArraySize: nullptr, Name: ReduceListArg->getName() + ".addr");
4408 ArrayType *RedListArrayTy =
4409 ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: ReductionInfos.size());
4410
4411 // 1. Build a list of reduction variables.
4412 // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
4413 Value *LocalReduceList =
4414 Builder.CreateAlloca(Ty: RedListArrayTy, ArraySize: nullptr, Name: ".omp.reduction.red_list");
4415
4416 InsertPointTy AllocaIP{EntryBlock, EntryBlock->begin()};
4417
4418 Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4419 V: BufferArgAlloca, DestTy: Builder.getPtrTy(),
4420 Name: BufferArgAlloca->getName() + ".ascast");
4421 Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4422 V: IdxArgAlloca, DestTy: Builder.getPtrTy(), Name: IdxArgAlloca->getName() + ".ascast");
4423 Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4424 V: ReduceListArgAlloca, DestTy: Builder.getPtrTy(),
4425 Name: ReduceListArgAlloca->getName() + ".ascast");
4426 Value *ReductionList = Builder.CreatePointerBitCastOrAddrSpaceCast(
4427 V: LocalReduceList, DestTy: Builder.getPtrTy(),
4428 Name: LocalReduceList->getName() + ".ascast");
4429
4430 Builder.CreateStore(Val: BufferArg, Ptr: BufferArgAddrCast);
4431 Builder.CreateStore(Val: IdxArg, Ptr: IdxArgAddrCast);
4432 Builder.CreateStore(Val: ReduceListArg, Ptr: ReduceListArgAddrCast);
4433
4434 Value *BufferVal = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: BufferArgAddrCast);
4435 Value *Idxs[] = {Builder.CreateLoad(Ty: Builder.getInt32Ty(), Ptr: IdxArgAddrCast)};
4436 Type *IndexTy = Builder.getIndexTy(
4437 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
4438 for (auto En : enumerate(First&: ReductionInfos)) {
4439 const ReductionInfo &RI = En.value();
4440
4441 Value *TargetElementPtrPtr = Builder.CreateInBoundsGEP(
4442 Ty: RedListArrayTy, Ptr: ReductionList,
4443 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
4444 // Global = Buffer.VD[Idx];
4445 Value *BufferVD =
4446 Builder.CreateInBoundsGEP(Ty: ReductionsBufferTy, Ptr: BufferVal, IdxList: Idxs);
4447 Value *GlobValPtr = Builder.CreateConstInBoundsGEP2_32(
4448 Ty: ReductionsBufferTy, Ptr: BufferVD, Idx0: 0, Idx1: En.index());
4449
4450 if (!IsByRef.empty() && IsByRef[En.index()] && RI.DataPtrPtrGen) {
4451 // Get source descriptor from the reduce list
4452 Value *ReduceListVal =
4453 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ReduceListArgAddrCast);
4454 Value *SrcElementPtrPtr =
4455 Builder.CreateInBoundsGEP(Ty: RedListArrayTy, Ptr: ReduceListVal,
4456 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0),
4457 ConstantInt::get(Ty: IndexTy, V: En.index())});
4458 Value *SrcDescriptorAddr =
4459 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: SrcElementPtrPtr);
4460
4461 // Copy descriptor from source and update base_ptr to global buffer data
4462 Expected<Value *> ByRefAlloc = createReductionDescriptorCopy(
4463 AllocaIP, RI, DataPtr: GlobValPtr, SrcDescriptorAddr, DescriptorPtrTy: Builder.getPtrTy());
4464 if (!ByRefAlloc)
4465 return ByRefAlloc.takeError();
4466
4467 Builder.CreateStore(Val: *ByRefAlloc, Ptr: TargetElementPtrPtr);
4468 } else {
4469 Builder.CreateStore(Val: GlobValPtr, Ptr: TargetElementPtrPtr);
4470 }
4471 }
4472
4473 // Call reduce_function(ReduceList, GlobalReduceList)
4474 Value *ReduceList =
4475 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ReduceListArgAddrCast);
4476 createRuntimeFunctionCall(Callee: ReduceFn, Args: {ReduceList, ReductionList})
4477 ->addFnAttr(Kind: Attribute::NoUnwind);
4478 Builder.CreateRetVoid();
4479 Builder.restoreIP(IP: OldIP);
4480 return GtLRFunc;
4481}
4482
4483std::string OpenMPIRBuilder::getReductionFuncName(StringRef Name) const {
4484 std::string Suffix =
4485 createPlatformSpecificName(Parts: {"omp", "reduction", "reduction_func"});
4486 return (Name + Suffix).str();
4487}
4488
4489Expected<Function *> OpenMPIRBuilder::createReductionFunction(
4490 StringRef ReducerName, ArrayRef<ReductionInfo> ReductionInfos,
4491 ArrayRef<bool> IsByRef, ReductionGenCBKind ReductionGenCBKind,
4492 AttributeList FuncAttrs) {
4493 auto *FuncTy = FunctionType::get(Result: Builder.getVoidTy(),
4494 Params: {Builder.getPtrTy(), Builder.getPtrTy()},
4495 /* IsVarArg */ isVarArg: false);
4496 std::string Name = getReductionFuncName(Name: ReducerName);
4497 Function *ReductionFunc =
4498 Function::Create(Ty: FuncTy, Linkage: GlobalVariable::InternalLinkage, N: Name, M: &M);
4499 ReductionFunc->setCallingConv(Config.getRuntimeCC());
4500 ReductionFunc->setAttributes(FuncAttrs);
4501 ReductionFunc->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
4502 ReductionFunc->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
4503 BasicBlock *EntryBB =
4504 BasicBlock::Create(Context&: M.getContext(), Name: "entry", Parent: ReductionFunc);
4505 Builder.SetInsertPoint(EntryBB);
4506
4507 // Need to alloca memory here and deal with the pointers before getting
4508 // LHS/RHS pointers out
4509 Value *LHSArrayPtr = nullptr;
4510 Value *RHSArrayPtr = nullptr;
4511 Argument *Arg0 = ReductionFunc->getArg(i: 0);
4512 Argument *Arg1 = ReductionFunc->getArg(i: 1);
4513 Type *Arg0Type = Arg0->getType();
4514 Type *Arg1Type = Arg1->getType();
4515
4516 Value *LHSAlloca =
4517 Builder.CreateAlloca(Ty: Arg0Type, ArraySize: nullptr, Name: Arg0->getName() + ".addr");
4518 Value *RHSAlloca =
4519 Builder.CreateAlloca(Ty: Arg1Type, ArraySize: nullptr, Name: Arg1->getName() + ".addr");
4520 Value *LHSAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4521 V: LHSAlloca, DestTy: Arg0Type, Name: LHSAlloca->getName() + ".ascast");
4522 Value *RHSAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4523 V: RHSAlloca, DestTy: Arg1Type, Name: RHSAlloca->getName() + ".ascast");
4524 Builder.CreateStore(Val: Arg0, Ptr: LHSAddrCast);
4525 Builder.CreateStore(Val: Arg1, Ptr: RHSAddrCast);
4526 LHSArrayPtr = Builder.CreateLoad(Ty: Arg0Type, Ptr: LHSAddrCast);
4527 RHSArrayPtr = Builder.CreateLoad(Ty: Arg1Type, Ptr: RHSAddrCast);
4528
4529 Type *RedArrayTy = ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: ReductionInfos.size());
4530 Type *IndexTy = Builder.getIndexTy(
4531 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
4532 SmallVector<Value *> LHSPtrs, RHSPtrs;
4533 for (auto En : enumerate(First&: ReductionInfos)) {
4534 const ReductionInfo &RI = En.value();
4535 Value *RHSI8PtrPtr = Builder.CreateInBoundsGEP(
4536 Ty: RedArrayTy, Ptr: RHSArrayPtr,
4537 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
4538 Value *RHSI8Ptr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: RHSI8PtrPtr);
4539 Value *RHSPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
4540 V: RHSI8Ptr, DestTy: RI.PrivateVariable->getType(),
4541 Name: RHSI8Ptr->getName() + ".ascast");
4542
4543 Value *LHSI8PtrPtr = Builder.CreateInBoundsGEP(
4544 Ty: RedArrayTy, Ptr: LHSArrayPtr,
4545 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
4546 Value *LHSI8Ptr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: LHSI8PtrPtr);
4547 Value *LHSPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
4548 V: LHSI8Ptr, DestTy: RI.Variable->getType(), Name: LHSI8Ptr->getName() + ".ascast");
4549
4550 if (ReductionGenCBKind == ReductionGenCBKind::Clang) {
4551 LHSPtrs.emplace_back(Args&: LHSPtr);
4552 RHSPtrs.emplace_back(Args&: RHSPtr);
4553 } else {
4554 Value *LHS = LHSPtr;
4555 Value *RHS = RHSPtr;
4556
4557 if (!IsByRef.empty() && !IsByRef[En.index()]) {
4558 LHS = Builder.CreateLoad(Ty: RI.ElementType, Ptr: LHSPtr);
4559 RHS = Builder.CreateLoad(Ty: RI.ElementType, Ptr: RHSPtr);
4560 }
4561
4562 Value *Reduced;
4563 InsertPointOrErrorTy AfterIP =
4564 RI.ReductionGen(Builder.saveIP(), LHS, RHS, Reduced);
4565 if (!AfterIP)
4566 return AfterIP.takeError();
4567 if (!Builder.GetInsertBlock())
4568 return ReductionFunc;
4569
4570 Builder.restoreIP(IP: *AfterIP);
4571
4572 if (!IsByRef.empty() && !IsByRef[En.index()])
4573 Builder.CreateStore(Val: Reduced, Ptr: LHSPtr);
4574 }
4575 }
4576
4577 if (ReductionGenCBKind == ReductionGenCBKind::Clang)
4578 for (auto En : enumerate(First&: ReductionInfos)) {
4579 unsigned Index = En.index();
4580 const ReductionInfo &RI = En.value();
4581 Value *LHSFixupPtr, *RHSFixupPtr;
4582 Builder.restoreIP(IP: RI.ReductionGenClang(
4583 Builder.saveIP(), Index, &LHSFixupPtr, &RHSFixupPtr, ReductionFunc));
4584
4585 // Fix the CallBack code genereated to use the correct Values for the LHS
4586 // and RHS
4587 LHSFixupPtr->replaceUsesWithIf(
4588 New: LHSPtrs[Index], ShouldReplace: [ReductionFunc](const Use &U) {
4589 return cast<Instruction>(Val: U.getUser())->getParent()->getParent() ==
4590 ReductionFunc;
4591 });
4592 RHSFixupPtr->replaceUsesWithIf(
4593 New: RHSPtrs[Index], ShouldReplace: [ReductionFunc](const Use &U) {
4594 return cast<Instruction>(Val: U.getUser())->getParent()->getParent() ==
4595 ReductionFunc;
4596 });
4597 }
4598
4599 Builder.CreateRetVoid();
4600 // Compiling with `-O0`, `alloca`s emitted in non-entry blocks are not hoisted
4601 // to the entry block (this is dones for higher opt levels by later passes in
4602 // the pipeline). This has caused issues because non-entry `alloca`s force the
4603 // function to use dynamic stack allocations and we might run out of scratch
4604 // memory.
4605 hoistNonEntryAllocasToEntryBlock(Func: ReductionFunc);
4606
4607 return ReductionFunc;
4608}
4609
4610static void
4611checkReductionInfos(ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos,
4612 bool IsGPU) {
4613 for (const OpenMPIRBuilder::ReductionInfo &RI : ReductionInfos) {
4614 (void)RI;
4615 assert(RI.Variable && "expected non-null variable");
4616 assert(RI.PrivateVariable && "expected non-null private variable");
4617 assert((RI.ReductionGen || RI.ReductionGenClang) &&
4618 "expected non-null reduction generator callback");
4619 if (!IsGPU) {
4620 assert(
4621 RI.Variable->getType() == RI.PrivateVariable->getType() &&
4622 "expected variables and their private equivalents to have the same "
4623 "type");
4624 }
4625 assert(RI.Variable->getType()->isPointerTy() &&
4626 "expected variables to be pointers");
4627 }
4628}
4629
4630OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createReductionsGPU(
4631 const LocationDescription &Loc, InsertPointTy AllocaIP,
4632 InsertPointTy CodeGenIP, ArrayRef<ReductionInfo> ReductionInfos,
4633 ArrayRef<bool> IsByRef, bool IsNoWait, bool IsTeamsReduction, bool IsSPMD,
4634 ReductionGenCBKind ReductionGenCBKind, std::optional<omp::GV> GridValue,
4635 Value *SrcLocInfo) {
4636 if (!updateToLocation(Loc))
4637 return InsertPointTy();
4638 Builder.restoreIP(IP: CodeGenIP);
4639 checkReductionInfos(ReductionInfos, /*IsGPU*/ true);
4640 LLVMContext &Ctx = M.getContext();
4641
4642 // Source location for the ident struct
4643 if (!SrcLocInfo) {
4644 uint32_t SrcLocStrSize;
4645 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
4646 SrcLocInfo = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
4647 }
4648
4649 if (ReductionInfos.size() == 0)
4650 return Builder.saveIP();
4651
4652 BasicBlock *ContinuationBlock = nullptr;
4653 if (ReductionGenCBKind != ReductionGenCBKind::Clang) {
4654 // Copied code from createReductions
4655 BasicBlock *InsertBlock = Loc.IP.getBlock();
4656 ContinuationBlock =
4657 InsertBlock->splitBasicBlock(I: Loc.IP.getPoint(), BBName: "reduce.finalize");
4658 InsertBlock->getTerminator()->eraseFromParent();
4659 Builder.SetInsertPoint(TheBB: InsertBlock, IP: InsertBlock->end());
4660 }
4661
4662 Function *CurFunc = Builder.GetInsertBlock()->getParent();
4663 AttributeList FuncAttrs;
4664 AttrBuilder AttrBldr(Ctx);
4665 for (auto Attr : CurFunc->getAttributes().getFnAttrs())
4666 AttrBldr.addAttribute(A: Attr);
4667 AttrBldr.removeAttribute(Val: Attribute::OptimizeNone);
4668 FuncAttrs = FuncAttrs.addFnAttributes(C&: Ctx, B: AttrBldr);
4669
4670 CodeGenIP = Builder.saveIP();
4671 Expected<Function *> ReductionResult = createReductionFunction(
4672 ReducerName: Builder.GetInsertBlock()->getParent()->getName(), ReductionInfos, IsByRef,
4673 ReductionGenCBKind, FuncAttrs);
4674 if (!ReductionResult)
4675 return ReductionResult.takeError();
4676 Function *ReductionFunc = *ReductionResult;
4677 Builder.restoreIP(IP: CodeGenIP);
4678
4679 // Set the grid value in the config needed for lowering later on
4680 if (GridValue.has_value())
4681 Config.setGridValue(GridValue.value());
4682 else
4683 Config.setGridValue(getGridValue(T, Kernel: ReductionFunc));
4684
4685 // Build res = __kmpc_reduce{_nowait}(<gtid>, <n>, sizeof(RedList),
4686 // RedList, shuffle_reduce_func, interwarp_copy_func);
4687 // or
4688 // Build res = __kmpc_reduce_teams_nowait_simple(<loc>, <gtid>, <lck>);
4689 Value *Res;
4690
4691 // 1. Build a list of reduction variables.
4692 // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
4693 auto Size = ReductionInfos.size();
4694 Type *PtrTy = PointerType::get(C&: Ctx, AddressSpace: Config.getDefaultTargetAS());
4695 Type *FuncPtrTy =
4696 Builder.getPtrTy(AddrSpace: M.getDataLayout().getProgramAddressSpace());
4697 Type *RedArrayTy = ArrayType::get(ElementType: PtrTy, NumElements: Size);
4698 CodeGenIP = Builder.saveIP();
4699 Builder.restoreIP(IP: AllocaIP);
4700 Value *ReductionListAlloca =
4701 Builder.CreateAlloca(Ty: RedArrayTy, ArraySize: nullptr, Name: ".omp.reduction.red_list");
4702 Value *ReductionList = Builder.CreatePointerBitCastOrAddrSpaceCast(
4703 V: ReductionListAlloca, DestTy: PtrTy, Name: ReductionListAlloca->getName() + ".ascast");
4704 Builder.restoreIP(IP: CodeGenIP);
4705 Type *IndexTy = Builder.getIndexTy(
4706 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
4707 for (auto En : enumerate(First&: ReductionInfos)) {
4708 const ReductionInfo &RI = En.value();
4709 Value *ElemPtr = Builder.CreateInBoundsGEP(
4710 Ty: RedArrayTy, Ptr: ReductionList,
4711 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
4712
4713 Value *PrivateVar = RI.PrivateVariable;
4714 bool IsByRefElem = !IsByRef.empty() && IsByRef[En.index()];
4715 if (IsByRefElem)
4716 PrivateVar = Builder.CreateLoad(Ty: RI.ElementType, Ptr: PrivateVar);
4717
4718 Value *CastElem =
4719 Builder.CreatePointerBitCastOrAddrSpaceCast(V: PrivateVar, DestTy: PtrTy);
4720 Builder.CreateStore(Val: CastElem, Ptr: ElemPtr);
4721 }
4722 CodeGenIP = Builder.saveIP();
4723 Expected<Function *> SarFunc = emitShuffleAndReduceFunction(
4724 ReductionInfos, ReduceFn: ReductionFunc, FuncAttrs, IsByRef);
4725
4726 if (!SarFunc)
4727 return SarFunc.takeError();
4728
4729 Expected<Function *> CopyResult =
4730 emitInterWarpCopyFunction(Loc, ReductionInfos, FuncAttrs, IsByRef);
4731 if (!CopyResult)
4732 return CopyResult.takeError();
4733 Function *WcFunc = *CopyResult;
4734 Builder.restoreIP(IP: CodeGenIP);
4735
4736 Value *RL = Builder.CreatePointerBitCastOrAddrSpaceCast(V: ReductionList, DestTy: PtrTy);
4737
4738 // NOTE: ReductionDataSize is passed as the reduce_data_size argument to
4739 // __kmpc_nvptx_parallel_reduce_nowait_v2, but the runtime implementations do
4740 // not currently use it. It is computed here conservatively as max(element
4741 // sizes) * N rather than the exact sum, which over-calculates the size for
4742 // mixed reduction types but is harmless given the argument is unused.
4743 // TODO: Consider dropping this computation if the runtime API is ever revised
4744 // to remove the unused parameter.
4745 unsigned MaxDataSize = 0;
4746 SmallVector<Type *> ReductionTypeArgs;
4747 for (auto En : enumerate(First&: ReductionInfos)) {
4748 // Use ByRefElementType for by-ref reductions so that MaxDataSize matches
4749 // the actual data size stored in the global reduction buffer, consistent
4750 // with the ReductionsBufferTy struct used for GEP offsets below.
4751 Type *RedTypeArg = (!IsByRef.empty() && IsByRef[En.index()])
4752 ? En.value().ByRefElementType
4753 : En.value().ElementType;
4754 auto Size = M.getDataLayout().getTypeStoreSize(Ty: RedTypeArg);
4755 if (Size > MaxDataSize)
4756 MaxDataSize = Size;
4757 ReductionTypeArgs.emplace_back(Args&: RedTypeArg);
4758 }
4759 Value *ReductionDataSize =
4760 Builder.getInt64(C: MaxDataSize * ReductionInfos.size());
4761
4762 // Helper function to copy thread-local data back to the original reduction
4763 // list.
4764 Function *CopyScratchToListFunc = nullptr;
4765 // Thread-local storage for the reduction variables.
4766 Value *ScratchForCopyBack = nullptr;
4767 // RL pointer to which the final value from the per-thread scratch should be
4768 // copied back. (Basically RL, appropriately casted if necessary.)
4769 Value *RLForCopyBack = RL;
4770
4771 if (!IsTeamsReduction) {
4772 Value *SarFuncCast =
4773 Builder.CreatePointerBitCastOrAddrSpaceCast(V: *SarFunc, DestTy: FuncPtrTy);
4774 Value *WcFuncCast =
4775 Builder.CreatePointerBitCastOrAddrSpaceCast(V: WcFunc, DestTy: FuncPtrTy);
4776 Value *Args[] = {SrcLocInfo, ReductionDataSize, RL, SarFuncCast,
4777 WcFuncCast};
4778 Function *Pv2Ptr = getOrCreateRuntimeFunctionPtr(
4779 FnID: RuntimeFunction::OMPRTL___kmpc_nvptx_parallel_reduce_nowait_v2);
4780 Res = createRuntimeFunctionCall(Callee: Pv2Ptr, Args);
4781 } else {
4782 CodeGenIP = Builder.saveIP();
4783 StructType *ReductionsBufferTy = StructType::create(
4784 Context&: Ctx, Elements: ReductionTypeArgs, Name: "struct._globalized_locals_ty");
4785
4786 Expected<Function *> LtGCFunc = emitListToGlobalCopyFunction(
4787 ReductionInfos, ReductionsBufferTy, FuncAttrs, IsByRef);
4788 if (!LtGCFunc)
4789 return LtGCFunc.takeError();
4790
4791 Expected<Function *> GtLCFunc = emitGlobalToListCopyFunction(
4792 ReductionInfos, ReductionsBufferTy, FuncAttrs, IsByRef);
4793 if (!GtLCFunc)
4794 return GtLCFunc.takeError();
4795
4796 Expected<Function *> GtLRFunc = emitGlobalToListReduceFunction(
4797 ReductionInfos, ReduceFn: ReductionFunc, ReductionsBufferTy, FuncAttrs, IsByRef);
4798 if (!GtLRFunc)
4799 return GtLRFunc.takeError();
4800
4801 Builder.restoreIP(IP: CodeGenIP);
4802
4803 // The runtime's cross-team final aggregate uses the storage pointed at by
4804 // its reduce-list argument as per-thread scratch. When the surrounding
4805 // kernel is already in SPMD execution mode, clang emitted each reduction
4806 // private as a per-thread `alloca addrspace(5)`, so the original red_list
4807 // (RL) is already per-thread and nothing else is needed.
4808 //
4809 // When the kernel is in Non-SPMD execution mode at codegen time, clang's
4810 // Generic-mode globalization put the reduction private into team-shared
4811 // LDS. OpenMPOpt may later upgrade the kernel to Generic-SPMD, at which
4812 // point all threads of the last team would race on the shared LDS slot.
4813 // Emit a per-thread scratch buffer and a per-thread RL, copy the team-local
4814 // value in, and hand the per-thread RL to the runtime instead. The writer
4815 // thread copies the final value from that per-thread scratch back to RL
4816 // before running the existing combine path below.
4817
4818 // Thread-local RL (might need localization below before being passed to the
4819 // runtime).
4820 Value *RuntimeRL = RL;
4821
4822 if (!IsSPMD) {
4823 CodeGenIP = Builder.saveIP();
4824 Builder.restoreIP(IP: AllocaIP);
4825 // Allocate thread-local buffer for the reduction variables.
4826 Value *PerThreadScratchAlloca = Builder.CreateAlloca(
4827 Ty: ReductionsBufferTy, /*ArraySize=*/nullptr, Name: ".omp.reduction.scratch");
4828 Value *PerThreadScratch = Builder.CreatePointerBitCastOrAddrSpaceCast(
4829 V: PerThreadScratchAlloca, DestTy: PtrTy,
4830 Name: PerThreadScratchAlloca->getName() + ".ascast");
4831 // Allocate thread-local buffer for the pointers to the reduction
4832 // variables.
4833 Value *PerThreadRedListAlloca =
4834 Builder.CreateAlloca(Ty: RedArrayTy, /*ArraySize=*/nullptr,
4835 Name: ".omp.reduction.per_thread_red_list");
4836 RuntimeRL = Builder.CreatePointerBitCastOrAddrSpaceCast(
4837 V: PerThreadRedListAlloca, DestTy: PtrTy,
4838 Name: PerThreadRedListAlloca->getName() + ".ascast");
4839 Builder.restoreIP(IP: CodeGenIP);
4840
4841 // Iterate over the reduction variables and copy the team-local value to
4842 // the thread-local buffer.
4843 for (auto En : enumerate(First&: ReductionInfos)) {
4844 const ReductionInfo &RI = En.value();
4845 bool IsByRefElem = !IsByRef.empty() && IsByRef[En.index()];
4846
4847 Value *FieldPtr = Builder.CreateConstInBoundsGEP2_32(
4848 Ty: ReductionsBufferTy, Ptr: PerThreadScratch, Idx0: 0, Idx1: En.index());
4849 Value *Slot = Builder.CreateConstInBoundsGEP2_32(Ty: RedArrayTy, Ptr: RuntimeRL,
4850 Idx0: 0, Idx1: En.index());
4851
4852 Value *RuntimeListEntry = FieldPtr;
4853 if (IsByRefElem && RI.DataPtrPtrGen) {
4854 Value *SrcDescriptor =
4855 Builder.CreateLoad(Ty: RI.ElementType, Ptr: RI.PrivateVariable);
4856 Expected<Value *> Descriptor = createReductionDescriptorCopy(
4857 AllocaIP, RI, DataPtr: FieldPtr, SrcDescriptorAddr: SrcDescriptor, DescriptorPtrTy: PtrTy);
4858 if (!Descriptor)
4859 return Descriptor.takeError();
4860 RuntimeListEntry = *Descriptor;
4861 }
4862 Builder.CreateStore(Val: RuntimeListEntry, Ptr: Slot);
4863 }
4864 // The copy helpers were emitted with default-AS (AS 0) pointer params
4865 // (see emitListToGlobalCopyFunction / emitGlobalToListCopyFunction),
4866 // but PerThreadScratch and RL live in the target's default AS, which
4867 // is non-zero on e.g. SPIRV. (See Config.getDefaultTargetAS().)
4868 Type *CopyArg0Ty = (*LtGCFunc)->getFunctionType()->getParamType(i: 0);
4869 Type *CopyArg2Ty = (*LtGCFunc)->getFunctionType()->getParamType(i: 2);
4870 ScratchForCopyBack = Builder.CreatePointerBitCastOrAddrSpaceCast(
4871 V: PerThreadScratch, DestTy: CopyArg0Ty);
4872 RLForCopyBack =
4873 Builder.CreatePointerBitCastOrAddrSpaceCast(V: RL, DestTy: CopyArg2Ty);
4874 // Use index 0 because there is no array of target values to index into,
4875 // there is only one thread-local memory slot.
4876 // restoreIP above left a stale/empty debug location; this inlinable call
4877 // to a debug-info-bearing helper needs one or the verifier rejects the
4878 // module ("!dbg attachment points at wrong subprogram") after inlining.
4879 Builder.SetCurrentDebugLocation(Loc.DL);
4880 Builder.CreateCall(
4881 Callee: *LtGCFunc, Args: {ScratchForCopyBack, Builder.getInt32(C: 0), RLForCopyBack});
4882 CopyScratchToListFunc = *GtLCFunc;
4883 }
4884
4885 Value *Args3[] = {SrcLocInfo, RuntimeRL, *SarFunc, WcFunc,
4886 *LtGCFunc, *GtLCFunc, *GtLRFunc};
4887
4888 Function *TeamsReduceFn = getOrCreateRuntimeFunctionPtr(
4889 FnID: RuntimeFunction::OMPRTL___kmpc_gpu_xteam_reduce_nowait);
4890 Res = createRuntimeFunctionCall(Callee: TeamsReduceFn, Args: Args3);
4891 }
4892
4893 // 5. Build if (res == 1)
4894 BasicBlock *ExitBB = BasicBlock::Create(Context&: Ctx, Name: ".omp.reduction.done");
4895 BasicBlock *ThenBB = BasicBlock::Create(Context&: Ctx, Name: ".omp.reduction.then");
4896 Value *Cond = Builder.CreateICmpEQ(LHS: Res, RHS: Builder.getInt32(C: 1));
4897 Builder.CreateCondBr(Cond, True: ThenBB, False: ExitBB);
4898
4899 // 6. Build then branch: where we have reduced values in the master
4900 // thread in each team.
4901 // __kmpc_end_reduce{_nowait}(<gtid>);
4902 // break;
4903 emitBlock(BB: ThenBB, CurFn: CurFunc);
4904
4905 // Copy the writer thread's per-thread scratch result back into the original
4906 // red-list storage before the existing combine path reads RI.PrivateVariable.
4907 // Set a debug location: this inlinable call to a debug-info-bearing helper
4908 // needs one or the verifier rejects the module after inlining.
4909 if (ScratchForCopyBack) {
4910 Builder.SetCurrentDebugLocation(Loc.DL);
4911 Builder.CreateCall(
4912 Callee: CopyScratchToListFunc,
4913 Args: {ScratchForCopyBack, Builder.getInt32(C: 0), RLForCopyBack});
4914 }
4915
4916 // Add emission of __kmpc_end_reduce{_nowait}(<gtid>);
4917 for (auto En : enumerate(First&: ReductionInfos)) {
4918 const ReductionInfo &RI = En.value();
4919 Type *ValueType = RI.ElementType;
4920 Value *RedValue = RI.Variable;
4921
4922 Value *RHS =
4923 Builder.CreatePointerBitCastOrAddrSpaceCast(V: RI.PrivateVariable, DestTy: PtrTy);
4924
4925 if (ReductionGenCBKind == ReductionGenCBKind::Clang) {
4926 Value *LHSPtr, *RHSPtr;
4927 Builder.restoreIP(IP: RI.ReductionGenClang(Builder.saveIP(), En.index(),
4928 &LHSPtr, &RHSPtr, CurFunc));
4929
4930 // Fix the CallBack code genereated to use the correct Values for the LHS
4931 // and RHS. Cast to match types before replacing (necessary to handle
4932 // different address spaces).
4933 if (LHSPtr->getType() != RedValue->getType())
4934 RedValue = Builder.CreatePointerBitCastOrAddrSpaceCast(
4935 V: RedValue, DestTy: LHSPtr->getType());
4936 if (RHSPtr->getType() != RHS->getType())
4937 RHS =
4938 Builder.CreatePointerBitCastOrAddrSpaceCast(V: RHS, DestTy: RHSPtr->getType());
4939
4940 LHSPtr->replaceUsesWithIf(New: RedValue, ShouldReplace: [ReductionFunc](const Use &U) {
4941 return cast<Instruction>(Val: U.getUser())->getParent()->getParent() ==
4942 ReductionFunc;
4943 });
4944 RHSPtr->replaceUsesWithIf(New: RHS, ShouldReplace: [ReductionFunc](const Use &U) {
4945 return cast<Instruction>(Val: U.getUser())->getParent()->getParent() ==
4946 ReductionFunc;
4947 });
4948 } else {
4949 if (IsByRef.empty() || !IsByRef[En.index()]) {
4950 RedValue = Builder.CreateLoad(Ty: ValueType, Ptr: RI.Variable,
4951 Name: "red.value." + Twine(En.index()));
4952 }
4953 Value *PrivateRedValue = Builder.CreateLoad(
4954 Ty: ValueType, Ptr: RHS, Name: "red.private.value" + Twine(En.index()));
4955 Value *Reduced;
4956 InsertPointOrErrorTy AfterIP =
4957 RI.ReductionGen(Builder.saveIP(), RedValue, PrivateRedValue, Reduced);
4958 if (!AfterIP)
4959 return AfterIP.takeError();
4960 Builder.restoreIP(IP: *AfterIP);
4961
4962 if (!IsByRef.empty() && !IsByRef[En.index()])
4963 Builder.CreateStore(Val: Reduced, Ptr: RI.Variable);
4964 }
4965 }
4966 emitBlock(BB: ExitBB, CurFn: CurFunc);
4967 if (ContinuationBlock) {
4968 Builder.CreateBr(Dest: ContinuationBlock);
4969 Builder.SetInsertPoint(ContinuationBlock);
4970 }
4971 Config.setEmitLLVMUsed();
4972
4973 return Builder.saveIP();
4974}
4975
4976static Function *getFreshReductionFunc(Module &M) {
4977 Type *VoidTy = Type::getVoidTy(C&: M.getContext());
4978 Type *Int8PtrTy = PointerType::getUnqual(C&: M.getContext());
4979 auto *FuncTy =
4980 FunctionType::get(Result: VoidTy, Params: {Int8PtrTy, Int8PtrTy}, /* IsVarArg */ isVarArg: false);
4981 return Function::Create(Ty: FuncTy, Linkage: GlobalVariable::InternalLinkage,
4982 N: ".omp.reduction.func", M: &M);
4983}
4984
4985static Error populateReductionFunction(
4986 Function *ReductionFunc,
4987 ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos,
4988 IRBuilder<> &Builder, ArrayRef<bool> IsByRef, bool IsGPU) {
4989 Module *Module = ReductionFunc->getParent();
4990 BasicBlock *ReductionFuncBlock =
4991 BasicBlock::Create(Context&: Module->getContext(), Name: "", Parent: ReductionFunc);
4992 Builder.SetInsertPoint(ReductionFuncBlock);
4993 Value *LHSArrayPtr = nullptr;
4994 Value *RHSArrayPtr = nullptr;
4995 if (IsGPU) {
4996 // Need to alloca memory here and deal with the pointers before getting
4997 // LHS/RHS pointers out
4998 //
4999 Argument *Arg0 = ReductionFunc->getArg(i: 0);
5000 Argument *Arg1 = ReductionFunc->getArg(i: 1);
5001 Type *Arg0Type = Arg0->getType();
5002 Type *Arg1Type = Arg1->getType();
5003
5004 Value *LHSAlloca =
5005 Builder.CreateAlloca(Ty: Arg0Type, ArraySize: nullptr, Name: Arg0->getName() + ".addr");
5006 Value *RHSAlloca =
5007 Builder.CreateAlloca(Ty: Arg1Type, ArraySize: nullptr, Name: Arg1->getName() + ".addr");
5008 Value *LHSAddrCast =
5009 Builder.CreatePointerBitCastOrAddrSpaceCast(V: LHSAlloca, DestTy: Arg0Type);
5010 Value *RHSAddrCast =
5011 Builder.CreatePointerBitCastOrAddrSpaceCast(V: RHSAlloca, DestTy: Arg1Type);
5012 Builder.CreateStore(Val: Arg0, Ptr: LHSAddrCast);
5013 Builder.CreateStore(Val: Arg1, Ptr: RHSAddrCast);
5014 LHSArrayPtr = Builder.CreateLoad(Ty: Arg0Type, Ptr: LHSAddrCast);
5015 RHSArrayPtr = Builder.CreateLoad(Ty: Arg1Type, Ptr: RHSAddrCast);
5016 } else {
5017 LHSArrayPtr = ReductionFunc->getArg(i: 0);
5018 RHSArrayPtr = ReductionFunc->getArg(i: 1);
5019 }
5020
5021 unsigned NumReductions = ReductionInfos.size();
5022 Type *RedArrayTy = ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: NumReductions);
5023
5024 for (auto En : enumerate(First&: ReductionInfos)) {
5025 const OpenMPIRBuilder::ReductionInfo &RI = En.value();
5026 Value *LHSI8PtrPtr = Builder.CreateConstInBoundsGEP2_64(
5027 Ty: RedArrayTy, Ptr: LHSArrayPtr, Idx0: 0, Idx1: En.index());
5028 Value *LHSI8Ptr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: LHSI8PtrPtr);
5029 Value *LHSPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
5030 V: LHSI8Ptr, DestTy: RI.Variable->getType());
5031 Value *LHS = Builder.CreateLoad(Ty: RI.ElementType, Ptr: LHSPtr);
5032 Value *RHSI8PtrPtr = Builder.CreateConstInBoundsGEP2_64(
5033 Ty: RedArrayTy, Ptr: RHSArrayPtr, Idx0: 0, Idx1: En.index());
5034 Value *RHSI8Ptr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: RHSI8PtrPtr);
5035 Value *RHSPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
5036 V: RHSI8Ptr, DestTy: RI.PrivateVariable->getType());
5037 Value *RHS = Builder.CreateLoad(Ty: RI.ElementType, Ptr: RHSPtr);
5038 Value *Reduced;
5039 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
5040 RI.ReductionGen(Builder.saveIP(), LHS, RHS, Reduced);
5041 if (!AfterIP)
5042 return AfterIP.takeError();
5043
5044 Builder.restoreIP(IP: *AfterIP);
5045 // TODO: Consider flagging an error.
5046 if (!Builder.GetInsertBlock())
5047 return Error::success();
5048
5049 // store is inside of the reduction region when using by-ref
5050 if (!IsByRef[En.index()])
5051 Builder.CreateStore(Val: Reduced, Ptr: LHSPtr);
5052 }
5053 Builder.CreateRetVoid();
5054 return Error::success();
5055}
5056
5057OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createReductions(
5058 const LocationDescription &Loc, InsertPointTy AllocaIP,
5059 ArrayRef<ReductionInfo> ReductionInfos, ArrayRef<bool> IsByRef,
5060 bool IsNoWait, bool IsTeamsReduction) {
5061 assert(ReductionInfos.size() == IsByRef.size());
5062 if (Config.isGPU())
5063 return createReductionsGPU(Loc, AllocaIP, CodeGenIP: Builder.saveIP(), ReductionInfos,
5064 IsByRef, IsNoWait, IsTeamsReduction);
5065
5066 checkReductionInfos(ReductionInfos, /*IsGPU*/ false);
5067
5068 if (!updateToLocation(Loc))
5069 return InsertPointTy();
5070
5071 if (ReductionInfos.size() == 0)
5072 return Builder.saveIP();
5073
5074 BasicBlock *InsertBlock = Loc.IP.getBlock();
5075 BasicBlock *ContinuationBlock =
5076 InsertBlock->splitBasicBlock(I: Loc.IP.getPoint(), BBName: "reduce.finalize");
5077 InsertBlock->getTerminator()->eraseFromParent();
5078
5079 // Create and populate array of type-erased pointers to private reduction
5080 // values.
5081 unsigned NumReductions = ReductionInfos.size();
5082 Type *RedArrayTy = ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: NumReductions);
5083 Builder.SetInsertPoint(AllocaIP.getBlock()->getTerminator());
5084 Value *RedArray = Builder.CreateAlloca(Ty: RedArrayTy, ArraySize: nullptr, Name: "red.array");
5085
5086 Builder.SetInsertPoint(TheBB: InsertBlock, IP: InsertBlock->end());
5087
5088 for (auto En : enumerate(First&: ReductionInfos)) {
5089 unsigned Index = En.index();
5090 const ReductionInfo &RI = En.value();
5091 Value *RedArrayElemPtr = Builder.CreateConstInBoundsGEP2_64(
5092 Ty: RedArrayTy, Ptr: RedArray, Idx0: 0, Idx1: Index, Name: "red.array.elem." + Twine(Index));
5093 Builder.CreateStore(Val: RI.PrivateVariable, Ptr: RedArrayElemPtr);
5094 }
5095
5096 // Emit a call to the runtime function that orchestrates the reduction.
5097 // Declare the reduction function in the process.
5098 Type *IndexTy = Builder.getIndexTy(
5099 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
5100 Function *Func = Builder.GetInsertBlock()->getParent();
5101 Module *Module = Func->getParent();
5102 uint32_t SrcLocStrSize;
5103 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
5104 bool CanGenerateAtomic = all_of(Range&: ReductionInfos, P: [](const ReductionInfo &RI) {
5105 return RI.AtomicReductionGen;
5106 });
5107 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize,
5108 LocFlags: CanGenerateAtomic
5109 ? IdentFlag::OMP_IDENT_FLAG_ATOMIC_REDUCE
5110 : IdentFlag(0));
5111 Value *ThreadId = getOrCreateThreadID(Ident);
5112 Constant *NumVariables = Builder.getInt32(C: NumReductions);
5113 const DataLayout &DL = Module->getDataLayout();
5114 unsigned RedArrayByteSize = DL.getTypeStoreSize(Ty: RedArrayTy);
5115 Constant *RedArraySize = ConstantInt::get(Ty: IndexTy, V: RedArrayByteSize);
5116 Function *ReductionFunc = getFreshReductionFunc(M&: *Module);
5117 Value *Lock = getOMPCriticalRegionLock(CriticalName: ".reduction");
5118 Function *ReduceFunc = getOrCreateRuntimeFunctionPtr(
5119 FnID: IsNoWait ? RuntimeFunction::OMPRTL___kmpc_reduce_nowait
5120 : RuntimeFunction::OMPRTL___kmpc_reduce);
5121 CallInst *ReduceCall =
5122 createRuntimeFunctionCall(Callee: ReduceFunc,
5123 Args: {Ident, ThreadId, NumVariables, RedArraySize,
5124 RedArray, ReductionFunc, Lock},
5125 Name: "reduce");
5126
5127 // Create final reduction entry blocks for the atomic and non-atomic case.
5128 // Emit IR that dispatches control flow to one of the blocks based on the
5129 // reduction supporting the atomic mode.
5130 BasicBlock *NonAtomicRedBlock =
5131 BasicBlock::Create(Context&: Module->getContext(), Name: "reduce.switch.nonatomic", Parent: Func);
5132 BasicBlock *AtomicRedBlock =
5133 BasicBlock::Create(Context&: Module->getContext(), Name: "reduce.switch.atomic", Parent: Func);
5134 SwitchInst *Switch =
5135 Builder.CreateSwitch(V: ReduceCall, Dest: ContinuationBlock, /* NumCases */ 2);
5136 Switch->addCase(OnVal: Builder.getInt32(C: 1), Dest: NonAtomicRedBlock);
5137 Switch->addCase(OnVal: Builder.getInt32(C: 2), Dest: AtomicRedBlock);
5138
5139 // Populate the non-atomic reduction using the elementwise reduction function.
5140 // This loads the elements from the global and private variables and reduces
5141 // them before storing back the result to the global variable.
5142 Builder.SetInsertPoint(NonAtomicRedBlock);
5143 for (auto En : enumerate(First&: ReductionInfos)) {
5144 const ReductionInfo &RI = En.value();
5145 Type *ValueType = RI.ElementType;
5146 // We have one less load for by-ref case because that load is now inside of
5147 // the reduction region
5148 Value *RedValue = RI.Variable;
5149 if (!IsByRef[En.index()]) {
5150 RedValue = Builder.CreateLoad(Ty: ValueType, Ptr: RI.Variable,
5151 Name: "red.value." + Twine(En.index()));
5152 }
5153 Value *PrivateRedValue =
5154 Builder.CreateLoad(Ty: ValueType, Ptr: RI.PrivateVariable,
5155 Name: "red.private.value." + Twine(En.index()));
5156 Value *Reduced;
5157 InsertPointOrErrorTy AfterIP =
5158 RI.ReductionGen(Builder.saveIP(), RedValue, PrivateRedValue, Reduced);
5159 if (!AfterIP)
5160 return AfterIP.takeError();
5161 Builder.restoreIP(IP: *AfterIP);
5162
5163 if (!Builder.GetInsertBlock())
5164 return InsertPointTy();
5165 // for by-ref case, the load is inside of the reduction region
5166 if (!IsByRef[En.index()])
5167 Builder.CreateStore(Val: Reduced, Ptr: RI.Variable);
5168 }
5169 Function *EndReduceFunc = getOrCreateRuntimeFunctionPtr(
5170 FnID: IsNoWait ? RuntimeFunction::OMPRTL___kmpc_end_reduce_nowait
5171 : RuntimeFunction::OMPRTL___kmpc_end_reduce);
5172 createRuntimeFunctionCall(Callee: EndReduceFunc, Args: {Ident, ThreadId, Lock});
5173 Builder.CreateBr(Dest: ContinuationBlock);
5174
5175 // Populate the atomic reduction using the atomic elementwise reduction
5176 // function. There are no loads/stores here because they will be happening
5177 // inside the atomic elementwise reduction.
5178 Builder.SetInsertPoint(AtomicRedBlock);
5179 if (CanGenerateAtomic && llvm::none_of(Range&: IsByRef, P: [](bool P) { return P; })) {
5180 for (const ReductionInfo &RI : ReductionInfos) {
5181 InsertPointOrErrorTy AfterIP = RI.AtomicReductionGen(
5182 Builder.saveIP(), RI.ElementType, RI.Variable, RI.PrivateVariable);
5183 if (!AfterIP)
5184 return AfterIP.takeError();
5185 Builder.restoreIP(IP: *AfterIP);
5186 if (!Builder.GetInsertBlock())
5187 return InsertPointTy();
5188 }
5189 Builder.CreateBr(Dest: ContinuationBlock);
5190 } else {
5191 Builder.CreateUnreachable();
5192 }
5193
5194 // Populate the outlined reduction function using the elementwise reduction
5195 // function. Partial values are extracted from the type-erased array of
5196 // pointers to private variables.
5197 Error Err = populateReductionFunction(ReductionFunc, ReductionInfos, Builder,
5198 IsByRef, /*isGPU=*/IsGPU: false);
5199 if (Err)
5200 return Err;
5201
5202 if (!Builder.GetInsertBlock())
5203 return InsertPointTy();
5204
5205 Builder.SetInsertPoint(ContinuationBlock);
5206 return Builder.saveIP();
5207}
5208
5209OpenMPIRBuilder::InsertPointOrErrorTy
5210OpenMPIRBuilder::createMaster(const LocationDescription &Loc,
5211 BodyGenCallbackTy BodyGenCB,
5212 FinalizeCallbackTy FiniCB) {
5213 if (!updateToLocation(Loc))
5214 return Loc.IP;
5215
5216 Directive OMPD = Directive::OMPD_master;
5217 uint32_t SrcLocStrSize;
5218 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
5219 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
5220 Value *ThreadId = getOrCreateThreadID(Ident);
5221 Value *Args[] = {Ident, ThreadId};
5222
5223 Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_master);
5224 Instruction *EntryCall = createRuntimeFunctionCall(Callee: EntryRTLFn, Args);
5225
5226 Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_end_master);
5227 Instruction *ExitCall = createRuntimeFunctionCall(Callee: ExitRTLFn, Args);
5228
5229 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
5230 /*Conditional*/ true, /*hasFinalize*/ HasFinalize: true);
5231}
5232
5233OpenMPIRBuilder::InsertPointOrErrorTy
5234OpenMPIRBuilder::createMasked(const LocationDescription &Loc,
5235 BodyGenCallbackTy BodyGenCB,
5236 FinalizeCallbackTy FiniCB, Value *Filter) {
5237 if (!updateToLocation(Loc))
5238 return Loc.IP;
5239
5240 Directive OMPD = Directive::OMPD_masked;
5241 uint32_t SrcLocStrSize;
5242 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
5243 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
5244 Value *ThreadId = getOrCreateThreadID(Ident);
5245 Value *Args[] = {Ident, ThreadId, Filter};
5246 Value *ArgsEnd[] = {Ident, ThreadId};
5247
5248 Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_masked);
5249 Instruction *EntryCall = createRuntimeFunctionCall(Callee: EntryRTLFn, Args);
5250
5251 Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_end_masked);
5252 Instruction *ExitCall = createRuntimeFunctionCall(Callee: ExitRTLFn, Args: ArgsEnd);
5253
5254 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
5255 /*Conditional*/ true, /*hasFinalize*/ HasFinalize: true);
5256}
5257
5258static llvm::CallInst *emitNoUnwindRuntimeCall(IRBuilder<> &Builder,
5259 llvm::FunctionCallee Callee,
5260 ArrayRef<llvm::Value *> Args,
5261 const llvm::Twine &Name) {
5262 llvm::CallInst *Call = Builder.CreateCall(
5263 Callee, Args, OpBundles: SmallVector<llvm::OperandBundleDef, 1>(), Name);
5264 Call->setDoesNotThrow();
5265 return Call;
5266}
5267
5268// Expects input basic block is dominated by BeforeScanBB.
5269// Once Scan directive is encountered, the code after scan directive should be
5270// dominated by AfterScanBB. Scan directive splits the code sequence to
5271// scan and input phase. Based on whether inclusive or exclusive
5272// clause is used in the scan directive and whether input loop or scan loop
5273// is lowered, it adds jumps to input and scan phase. First Scan loop is the
5274// input loop and second is the scan loop. The code generated handles only
5275// inclusive scans now.
5276OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createScan(
5277 const LocationDescription &Loc, InsertPointTy AllocaIP,
5278 ArrayRef<llvm::Value *> ScanVars, ArrayRef<llvm::Type *> ScanVarsType,
5279 bool IsInclusive, ScanInfo *ScanRedInfo) {
5280 if (ScanRedInfo->OMPFirstScanLoop) {
5281 llvm::Error Err = emitScanBasedDirectiveDeclsIR(AllocaIP, ScanVars,
5282 ScanVarsType, ScanRedInfo);
5283 if (Err)
5284 return Err;
5285 }
5286 if (!updateToLocation(Loc))
5287 return Loc.IP;
5288
5289 llvm::Value *IV = ScanRedInfo->IV;
5290
5291 if (ScanRedInfo->OMPFirstScanLoop) {
5292 // Emit buffer[i] = red; at the end of the input phase.
5293 for (size_t i = 0; i < ScanVars.size(); i++) {
5294 Value *BuffPtr = (*(ScanRedInfo->ScanBuffPtrs))[ScanVars[i]];
5295 Value *Buff = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: BuffPtr);
5296 Type *DestTy = ScanVarsType[i];
5297 Value *Val = Builder.CreateInBoundsGEP(Ty: DestTy, Ptr: Buff, IdxList: IV, Name: "arrayOffset");
5298 Value *Src = Builder.CreateLoad(Ty: DestTy, Ptr: ScanVars[i]);
5299
5300 Builder.CreateStore(Val: Src, Ptr: Val);
5301 }
5302 }
5303 Builder.CreateBr(Dest: ScanRedInfo->OMPScanLoopExit);
5304 emitBlock(BB: ScanRedInfo->OMPScanDispatch,
5305 CurFn: Builder.GetInsertBlock()->getParent());
5306
5307 if (!ScanRedInfo->OMPFirstScanLoop) {
5308 IV = ScanRedInfo->IV;
5309 // Emit red = buffer[i]; at the entrance to the scan phase.
5310 // TODO: if exclusive scan, the red = buffer[i-1] needs to be updated.
5311 for (size_t i = 0; i < ScanVars.size(); i++) {
5312 Value *BuffPtr = (*(ScanRedInfo->ScanBuffPtrs))[ScanVars[i]];
5313 Value *Buff = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: BuffPtr);
5314 Type *DestTy = ScanVarsType[i];
5315 Value *SrcPtr =
5316 Builder.CreateInBoundsGEP(Ty: DestTy, Ptr: Buff, IdxList: IV, Name: "arrayOffset");
5317 Value *Src = Builder.CreateLoad(Ty: DestTy, Ptr: SrcPtr);
5318 Builder.CreateStore(Val: Src, Ptr: ScanVars[i]);
5319 }
5320 }
5321
5322 // TODO: Update it to CreateBr and remove dead blocks
5323 llvm::Value *CmpI = Builder.getInt1(V: true);
5324 if (ScanRedInfo->OMPFirstScanLoop == IsInclusive) {
5325 Builder.CreateCondBr(Cond: CmpI, True: ScanRedInfo->OMPBeforeScanBlock,
5326 False: ScanRedInfo->OMPAfterScanBlock);
5327 } else {
5328 Builder.CreateCondBr(Cond: CmpI, True: ScanRedInfo->OMPAfterScanBlock,
5329 False: ScanRedInfo->OMPBeforeScanBlock);
5330 }
5331 emitBlock(BB: ScanRedInfo->OMPAfterScanBlock,
5332 CurFn: Builder.GetInsertBlock()->getParent());
5333 Builder.SetInsertPoint(ScanRedInfo->OMPAfterScanBlock);
5334 return Builder.saveIP();
5335}
5336
5337Error OpenMPIRBuilder::emitScanBasedDirectiveDeclsIR(
5338 InsertPointTy AllocaIP, ArrayRef<Value *> ScanVars,
5339 ArrayRef<Type *> ScanVarsType, ScanInfo *ScanRedInfo) {
5340
5341 Builder.restoreIP(IP: AllocaIP);
5342 // Create the shared pointer at alloca IP.
5343 for (size_t i = 0; i < ScanVars.size(); i++) {
5344 llvm::Value *BuffPtr =
5345 Builder.CreateAlloca(Ty: Builder.getPtrTy(), ArraySize: nullptr, Name: "vla");
5346 (*(ScanRedInfo->ScanBuffPtrs))[ScanVars[i]] = BuffPtr;
5347 }
5348
5349 // Allocate temporary buffer by master thread
5350 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
5351 ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
5352 Builder.restoreIP(IP: CodeGenIP);
5353 Value *AllocSpan =
5354 Builder.CreateAdd(LHS: ScanRedInfo->Span, RHS: Builder.getInt32(C: 1));
5355 for (size_t i = 0; i < ScanVars.size(); i++) {
5356 Type *IntPtrTy = Builder.getInt32Ty();
5357 Constant *Allocsize = ConstantExpr::getSizeOf(Ty: ScanVarsType[i]);
5358 Allocsize = ConstantExpr::getTruncOrBitCast(C: Allocsize, Ty: IntPtrTy);
5359 Value *Buff = Builder.CreateMalloc(IntPtrTy, AllocTy: ScanVarsType[i], AllocSize: Allocsize,
5360 ArraySize: AllocSpan, MallocF: nullptr, Name: "arr");
5361 Builder.CreateStore(Val: Buff, Ptr: (*(ScanRedInfo->ScanBuffPtrs))[ScanVars[i]]);
5362 }
5363 return Error::success();
5364 };
5365 // TODO: Perform finalization actions for variables. This has to be
5366 // called for variables which have destructors/finalizers.
5367 auto FiniCB = [&](InsertPointTy CodeGenIP) { return llvm::Error::success(); };
5368
5369 Builder.SetInsertPoint(ScanRedInfo->OMPScanInit->getTerminator());
5370 llvm::Value *FilterVal = Builder.getInt32(C: 0);
5371 llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
5372 createMasked(Loc: Builder.saveIP(), BodyGenCB, FiniCB, Filter: FilterVal);
5373
5374 if (!AfterIP)
5375 return AfterIP.takeError();
5376 Builder.restoreIP(IP: *AfterIP);
5377 BasicBlock *InputBB = Builder.GetInsertBlock();
5378 if (InputBB->hasTerminator())
5379 Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator());
5380 AfterIP = createBarrier(Loc: Builder.saveIP(), Kind: llvm::omp::OMPD_barrier);
5381 if (!AfterIP)
5382 return AfterIP.takeError();
5383 Builder.restoreIP(IP: *AfterIP);
5384
5385 return Error::success();
5386}
5387
5388Error OpenMPIRBuilder::emitScanBasedDirectiveFinalsIR(
5389 ArrayRef<ReductionInfo> ReductionInfos, ScanInfo *ScanRedInfo) {
5390 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
5391 ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
5392 Builder.restoreIP(IP: CodeGenIP);
5393 for (ReductionInfo RedInfo : ReductionInfos) {
5394 Value *PrivateVar = RedInfo.PrivateVariable;
5395 Value *OrigVar = RedInfo.Variable;
5396 Value *BuffPtr = (*(ScanRedInfo->ScanBuffPtrs))[PrivateVar];
5397 Value *Buff = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: BuffPtr);
5398
5399 Type *SrcTy = RedInfo.ElementType;
5400 Value *Val = Builder.CreateInBoundsGEP(Ty: SrcTy, Ptr: Buff, IdxList: ScanRedInfo->Span,
5401 Name: "arrayOffset");
5402 Value *Src = Builder.CreateLoad(Ty: SrcTy, Ptr: Val);
5403
5404 Builder.CreateStore(Val: Src, Ptr: OrigVar);
5405 Builder.CreateFree(Source: Buff);
5406 }
5407 return Error::success();
5408 };
5409 // TODO: Perform finalization actions for variables. This has to be
5410 // called for variables which have destructors/finalizers.
5411 auto FiniCB = [&](InsertPointTy CodeGenIP) { return llvm::Error::success(); };
5412
5413 if (Instruction *TI = ScanRedInfo->OMPScanFinish->getTerminatorOrNull())
5414 Builder.SetInsertPoint(TI);
5415 else
5416 Builder.SetInsertPoint(ScanRedInfo->OMPScanFinish);
5417
5418 llvm::Value *FilterVal = Builder.getInt32(C: 0);
5419 llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
5420 createMasked(Loc: Builder.saveIP(), BodyGenCB, FiniCB, Filter: FilterVal);
5421
5422 if (!AfterIP)
5423 return AfterIP.takeError();
5424 Builder.restoreIP(IP: *AfterIP);
5425 BasicBlock *InputBB = Builder.GetInsertBlock();
5426 if (InputBB->hasTerminator())
5427 Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator());
5428 AfterIP = createBarrier(Loc: Builder.saveIP(), Kind: llvm::omp::OMPD_barrier);
5429 if (!AfterIP)
5430 return AfterIP.takeError();
5431 Builder.restoreIP(IP: *AfterIP);
5432 return Error::success();
5433}
5434
5435OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
5436 const LocationDescription &Loc,
5437 ArrayRef<llvm::OpenMPIRBuilder::ReductionInfo> ReductionInfos,
5438 ScanInfo *ScanRedInfo) {
5439
5440 if (!updateToLocation(Loc))
5441 return Loc.IP;
5442 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
5443 ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
5444 Builder.restoreIP(IP: CodeGenIP);
5445 Function *CurFn = Builder.GetInsertBlock()->getParent();
5446 // for (int k = 0; k <= ceil(log2(n)); ++k)
5447 llvm::BasicBlock *LoopBB =
5448 BasicBlock::Create(Context&: CurFn->getContext(), Name: "omp.outer.log.scan.body");
5449 llvm::BasicBlock *ExitBB =
5450 splitBB(Builder, CreateBranch: false, Name: "omp.outer.log.scan.exit");
5451 llvm::Function *F = llvm::Intrinsic::getOrInsertDeclaration(
5452 M: Builder.GetInsertBlock()->getModule(),
5453 id: (llvm::Intrinsic::ID)llvm::Intrinsic::log2, OverloadTys: Builder.getDoubleTy());
5454 llvm::BasicBlock *InputBB = Builder.GetInsertBlock();
5455 llvm::Value *Arg =
5456 Builder.CreateUIToFP(V: ScanRedInfo->Span, DestTy: Builder.getDoubleTy());
5457 llvm::Value *LogVal = emitNoUnwindRuntimeCall(Builder, Callee: F, Args: Arg, Name: "");
5458 F = llvm::Intrinsic::getOrInsertDeclaration(
5459 M: Builder.GetInsertBlock()->getModule(),
5460 id: (llvm::Intrinsic::ID)llvm::Intrinsic::ceil, OverloadTys: Builder.getDoubleTy());
5461 LogVal = emitNoUnwindRuntimeCall(Builder, Callee: F, Args: LogVal, Name: "");
5462 LogVal = Builder.CreateFPToUI(V: LogVal, DestTy: Builder.getInt32Ty());
5463 llvm::Value *NMin1 = Builder.CreateNUWSub(
5464 LHS: ScanRedInfo->Span,
5465 RHS: llvm::ConstantInt::get(Ty: ScanRedInfo->Span->getType(), V: 1));
5466 Builder.SetInsertPoint(InputBB);
5467 Builder.CreateBr(Dest: LoopBB);
5468 emitBlock(BB: LoopBB, CurFn);
5469 Builder.SetInsertPoint(LoopBB);
5470
5471 PHINode *Counter = Builder.CreatePHI(Ty: Builder.getInt32Ty(), NumReservedValues: 2);
5472 // size pow2k = 1;
5473 PHINode *Pow2K = Builder.CreatePHI(Ty: Builder.getInt32Ty(), NumReservedValues: 2);
5474 Counter->addIncoming(V: llvm::ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0),
5475 BB: InputBB);
5476 Pow2K->addIncoming(V: llvm::ConstantInt::get(Ty: Builder.getInt32Ty(), V: 1),
5477 BB: InputBB);
5478 // for (size i = n - 1; i >= 2 ^ k; --i)
5479 // tmp[i] op= tmp[i-pow2k];
5480 llvm::BasicBlock *InnerLoopBB =
5481 BasicBlock::Create(Context&: CurFn->getContext(), Name: "omp.inner.log.scan.body");
5482 llvm::BasicBlock *InnerExitBB =
5483 BasicBlock::Create(Context&: CurFn->getContext(), Name: "omp.inner.log.scan.exit");
5484 llvm::Value *CmpI = Builder.CreateICmpUGE(LHS: NMin1, RHS: Pow2K);
5485 Builder.CreateCondBr(Cond: CmpI, True: InnerLoopBB, False: InnerExitBB);
5486 emitBlock(BB: InnerLoopBB, CurFn);
5487 Builder.SetInsertPoint(InnerLoopBB);
5488 PHINode *IVal = Builder.CreatePHI(Ty: Builder.getInt32Ty(), NumReservedValues: 2);
5489 IVal->addIncoming(V: NMin1, BB: LoopBB);
5490 for (ReductionInfo RedInfo : ReductionInfos) {
5491 Value *ReductionVal = RedInfo.PrivateVariable;
5492 Value *BuffPtr = (*(ScanRedInfo->ScanBuffPtrs))[ReductionVal];
5493 Value *Buff = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: BuffPtr);
5494 Type *DestTy = RedInfo.ElementType;
5495 Value *IV = Builder.CreateAdd(LHS: IVal, RHS: Builder.getInt32(C: 1));
5496 Value *LHSPtr =
5497 Builder.CreateInBoundsGEP(Ty: DestTy, Ptr: Buff, IdxList: IV, Name: "arrayOffset");
5498 Value *OffsetIval = Builder.CreateNUWSub(LHS: IV, RHS: Pow2K);
5499 Value *RHSPtr =
5500 Builder.CreateInBoundsGEP(Ty: DestTy, Ptr: Buff, IdxList: OffsetIval, Name: "arrayOffset");
5501 Value *LHS = Builder.CreateLoad(Ty: DestTy, Ptr: LHSPtr);
5502 Value *RHS = Builder.CreateLoad(Ty: DestTy, Ptr: RHSPtr);
5503 llvm::Value *Result;
5504 InsertPointOrErrorTy AfterIP =
5505 RedInfo.ReductionGen(Builder.saveIP(), LHS, RHS, Result);
5506 if (!AfterIP)
5507 return AfterIP.takeError();
5508 Builder.CreateStore(Val: Result, Ptr: LHSPtr);
5509 }
5510 llvm::Value *NextIVal = Builder.CreateNUWSub(
5511 LHS: IVal, RHS: llvm::ConstantInt::get(Ty: Builder.getInt32Ty(), V: 1));
5512 IVal->addIncoming(V: NextIVal, BB: Builder.GetInsertBlock());
5513 CmpI = Builder.CreateICmpUGE(LHS: NextIVal, RHS: Pow2K);
5514 Builder.CreateCondBr(Cond: CmpI, True: InnerLoopBB, False: InnerExitBB);
5515 emitBlock(BB: InnerExitBB, CurFn);
5516 llvm::Value *Next = Builder.CreateNUWAdd(
5517 LHS: Counter, RHS: llvm::ConstantInt::get(Ty: Counter->getType(), V: 1));
5518 Counter->addIncoming(V: Next, BB: Builder.GetInsertBlock());
5519 // pow2k <<= 1;
5520 llvm::Value *NextPow2K = Builder.CreateShl(LHS: Pow2K, RHS: 1, Name: "", /*HasNUW=*/true);
5521 Pow2K->addIncoming(V: NextPow2K, BB: Builder.GetInsertBlock());
5522 llvm::Value *Cmp = Builder.CreateICmpNE(LHS: Next, RHS: LogVal);
5523 Builder.CreateCondBr(Cond: Cmp, True: LoopBB, False: ExitBB);
5524 Builder.SetInsertPoint(ExitBB->getFirstInsertionPt());
5525 return Error::success();
5526 };
5527
5528 // TODO: Perform finalization actions for variables. This has to be
5529 // called for variables which have destructors/finalizers.
5530 auto FiniCB = [&](InsertPointTy CodeGenIP) { return llvm::Error::success(); };
5531
5532 llvm::Value *FilterVal = Builder.getInt32(C: 0);
5533 llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
5534 createMasked(Loc: Builder.saveIP(), BodyGenCB, FiniCB, Filter: FilterVal);
5535
5536 if (!AfterIP)
5537 return AfterIP.takeError();
5538 Builder.restoreIP(IP: *AfterIP);
5539 AfterIP = createBarrier(Loc: Builder.saveIP(), Kind: llvm::omp::OMPD_barrier);
5540
5541 if (!AfterIP)
5542 return AfterIP.takeError();
5543 Builder.restoreIP(IP: *AfterIP);
5544 Error Err = emitScanBasedDirectiveFinalsIR(ReductionInfos, ScanRedInfo);
5545 if (Err)
5546 return Err;
5547
5548 return AfterIP;
5549}
5550
5551Error OpenMPIRBuilder::emitScanBasedDirectiveIR(
5552 llvm::function_ref<Error()> InputLoopGen,
5553 llvm::function_ref<Error(LocationDescription Loc)> ScanLoopGen,
5554 ScanInfo *ScanRedInfo) {
5555
5556 {
5557 // Emit loop with input phase:
5558 // for (i: 0..<num_iters>) {
5559 // <input phase>;
5560 // buffer[i] = red;
5561 // }
5562 ScanRedInfo->OMPFirstScanLoop = true;
5563 Error Err = InputLoopGen();
5564 if (Err)
5565 return Err;
5566 }
5567 {
5568 // Emit loop with scan phase:
5569 // for (i: 0..<num_iters>) {
5570 // red = buffer[i];
5571 // <scan phase>;
5572 // }
5573 ScanRedInfo->OMPFirstScanLoop = false;
5574 Error Err = ScanLoopGen(Builder.saveIP());
5575 if (Err)
5576 return Err;
5577 }
5578 return Error::success();
5579}
5580
5581void OpenMPIRBuilder::createScanBBs(ScanInfo *ScanRedInfo) {
5582 Function *Fun = Builder.GetInsertBlock()->getParent();
5583 ScanRedInfo->OMPScanDispatch =
5584 BasicBlock::Create(Context&: Fun->getContext(), Name: "omp.inscan.dispatch");
5585 ScanRedInfo->OMPAfterScanBlock =
5586 BasicBlock::Create(Context&: Fun->getContext(), Name: "omp.after.scan.bb");
5587 ScanRedInfo->OMPBeforeScanBlock =
5588 BasicBlock::Create(Context&: Fun->getContext(), Name: "omp.before.scan.bb");
5589 ScanRedInfo->OMPScanLoopExit =
5590 BasicBlock::Create(Context&: Fun->getContext(), Name: "omp.scan.loop.exit");
5591}
5592CanonicalLoopInfo *OpenMPIRBuilder::createLoopSkeleton(
5593 DebugLoc DL, Value *TripCount, Function *F, BasicBlock *PreInsertBefore,
5594 BasicBlock *PostInsertBefore, const Twine &Name) {
5595 Module *M = F->getParent();
5596 LLVMContext &Ctx = M->getContext();
5597 Type *IndVarTy = TripCount->getType();
5598
5599 // Create the basic block structure.
5600 BasicBlock *Preheader =
5601 BasicBlock::Create(Context&: Ctx, Name: "omp_" + Name + ".preheader", Parent: F, InsertBefore: PreInsertBefore);
5602 BasicBlock *Header =
5603 BasicBlock::Create(Context&: Ctx, Name: "omp_" + Name + ".header", Parent: F, InsertBefore: PreInsertBefore);
5604 BasicBlock *Cond =
5605 BasicBlock::Create(Context&: Ctx, Name: "omp_" + Name + ".cond", Parent: F, InsertBefore: PreInsertBefore);
5606 BasicBlock *Body =
5607 BasicBlock::Create(Context&: Ctx, Name: "omp_" + Name + ".body", Parent: F, InsertBefore: PreInsertBefore);
5608 BasicBlock *Latch =
5609 BasicBlock::Create(Context&: Ctx, Name: "omp_" + Name + ".inc", Parent: F, InsertBefore: PostInsertBefore);
5610 BasicBlock *Exit =
5611 BasicBlock::Create(Context&: Ctx, Name: "omp_" + Name + ".exit", Parent: F, InsertBefore: PostInsertBefore);
5612 BasicBlock *After =
5613 BasicBlock::Create(Context&: Ctx, Name: "omp_" + Name + ".after", Parent: F, InsertBefore: PostInsertBefore);
5614
5615 // Use specified DebugLoc for new instructions.
5616 Builder.SetCurrentDebugLocation(DL);
5617
5618 Builder.SetInsertPoint(Preheader);
5619 Builder.CreateBr(Dest: Header);
5620
5621 Builder.SetInsertPoint(Header);
5622 PHINode *IndVarPHI = Builder.CreatePHI(Ty: IndVarTy, NumReservedValues: 2, Name: "omp_" + Name + ".iv");
5623 IndVarPHI->addIncoming(V: ConstantInt::get(Ty: IndVarTy, V: 0), BB: Preheader);
5624 Builder.CreateBr(Dest: Cond);
5625
5626 Builder.SetInsertPoint(Cond);
5627 Value *Cmp =
5628 Builder.CreateICmpULT(LHS: IndVarPHI, RHS: TripCount, Name: "omp_" + Name + ".cmp");
5629 Builder.CreateCondBr(Cond: Cmp, True: Body, False: Exit);
5630
5631 Builder.SetInsertPoint(Body);
5632 Builder.CreateBr(Dest: Latch);
5633
5634 Builder.SetInsertPoint(Latch);
5635 Value *Next = Builder.CreateAdd(LHS: IndVarPHI, RHS: ConstantInt::get(Ty: IndVarTy, V: 1),
5636 Name: "omp_" + Name + ".next", /*HasNUW=*/true);
5637 Builder.CreateBr(Dest: Header);
5638 IndVarPHI->addIncoming(V: Next, BB: Latch);
5639
5640 Builder.SetInsertPoint(Exit);
5641 Builder.CreateBr(Dest: After);
5642
5643 // Remember and return the canonical control flow.
5644 LoopInfos.emplace_front();
5645 CanonicalLoopInfo *CL = &LoopInfos.front();
5646
5647 CL->Header = Header;
5648 CL->Cond = Cond;
5649 CL->Latch = Latch;
5650 CL->Exit = Exit;
5651
5652#ifndef NDEBUG
5653 CL->assertOK();
5654#endif
5655 return CL;
5656}
5657
5658Expected<CanonicalLoopInfo *>
5659OpenMPIRBuilder::createCanonicalLoop(const LocationDescription &Loc,
5660 LoopBodyGenCallbackTy BodyGenCB,
5661 Value *TripCount, const Twine &Name) {
5662 BasicBlock *BB = Loc.IP.getBlock();
5663 BasicBlock *NextBB = BB->getNextNode();
5664
5665 CanonicalLoopInfo *CL = createLoopSkeleton(DL: Loc.DL, TripCount, F: BB->getParent(),
5666 PreInsertBefore: NextBB, PostInsertBefore: NextBB, Name);
5667 BasicBlock *After = CL->getAfter();
5668
5669 // If location is not set, don't connect the loop.
5670 if (updateToLocation(Loc)) {
5671 // Split the loop at the insertion point: Branch to the preheader and move
5672 // every following instruction to after the loop (the After BB). Also, the
5673 // new successor is the loop's after block.
5674 spliceBB(Builder, New: After, /*CreateBranch=*/false);
5675 Builder.CreateBr(Dest: CL->getPreheader());
5676 }
5677
5678 // Emit the body content. We do it after connecting the loop to the CFG to
5679 // avoid that the callback encounters degenerate BBs.
5680 if (Error Err = BodyGenCB(CL->getBodyIP(), CL->getIndVar()))
5681 return Err;
5682
5683#ifndef NDEBUG
5684 CL->assertOK();
5685#endif
5686 return CL;
5687}
5688
5689Expected<ScanInfo *> OpenMPIRBuilder::scanInfoInitialize() {
5690 ScanInfos.emplace_front();
5691 ScanInfo *Result = &ScanInfos.front();
5692 return Result;
5693}
5694
5695Expected<SmallVector<llvm::CanonicalLoopInfo *>>
5696OpenMPIRBuilder::createCanonicalScanLoops(
5697 const LocationDescription &Loc, LoopBodyGenCallbackTy BodyGenCB,
5698 Value *Start, Value *Stop, Value *Step, bool IsSigned, bool InclusiveStop,
5699 InsertPointTy ComputeIP, const Twine &Name, ScanInfo *ScanRedInfo) {
5700 LocationDescription ComputeLoc =
5701 ComputeIP.isSet() ? LocationDescription(ComputeIP, Loc.DL) : Loc;
5702 updateToLocation(Loc: ComputeLoc);
5703
5704 SmallVector<CanonicalLoopInfo *> Result;
5705
5706 Value *TripCount = calculateCanonicalLoopTripCount(
5707 Loc: ComputeLoc, Start, Stop, Step, IsSigned, InclusiveStop, Name);
5708 ScanRedInfo->Span = TripCount;
5709 ScanRedInfo->OMPScanInit = splitBB(Builder, CreateBranch: true, Name: "scan.init");
5710 Builder.SetInsertPoint(ScanRedInfo->OMPScanInit);
5711
5712 auto BodyGen = [=](InsertPointTy CodeGenIP, Value *IV) {
5713 Builder.restoreIP(IP: CodeGenIP);
5714 ScanRedInfo->IV = IV;
5715 createScanBBs(ScanRedInfo);
5716 BasicBlock *InputBlock = Builder.GetInsertBlock();
5717 Instruction *Terminator = InputBlock->getTerminator();
5718 assert(Terminator->getNumSuccessors() == 1);
5719 BasicBlock *ContinueBlock = Terminator->getSuccessor(Idx: 0);
5720 Terminator->setSuccessor(Idx: 0, BB: ScanRedInfo->OMPScanDispatch);
5721 emitBlock(BB: ScanRedInfo->OMPBeforeScanBlock,
5722 CurFn: Builder.GetInsertBlock()->getParent());
5723 Builder.CreateBr(Dest: ScanRedInfo->OMPScanLoopExit);
5724 emitBlock(BB: ScanRedInfo->OMPScanLoopExit,
5725 CurFn: Builder.GetInsertBlock()->getParent());
5726 Builder.CreateBr(Dest: ContinueBlock);
5727 Builder.SetInsertPoint(
5728 ScanRedInfo->OMPBeforeScanBlock->getFirstInsertionPt());
5729 return BodyGenCB(Builder.saveIP(), IV);
5730 };
5731
5732 const auto &&InputLoopGen = [&]() -> Error {
5733 Expected<CanonicalLoopInfo *> LoopInfo = createCanonicalLoop(
5734 Loc: Builder.saveIP(), BodyGenCB: BodyGen, Start, Stop, Step, IsSigned, InclusiveStop,
5735 ComputeIP, Name, InScan: true, ScanRedInfo);
5736 if (!LoopInfo)
5737 return LoopInfo.takeError();
5738 Result.push_back(Elt: *LoopInfo);
5739 Builder.restoreIP(IP: (*LoopInfo)->getAfterIP());
5740 return Error::success();
5741 };
5742 const auto &&ScanLoopGen = [&](LocationDescription Loc) -> Error {
5743 Expected<CanonicalLoopInfo *> LoopInfo =
5744 createCanonicalLoop(Loc, BodyGenCB: BodyGen, Start, Stop, Step, IsSigned,
5745 InclusiveStop, ComputeIP, Name, InScan: true, ScanRedInfo);
5746 if (!LoopInfo)
5747 return LoopInfo.takeError();
5748 Result.push_back(Elt: *LoopInfo);
5749 Builder.restoreIP(IP: (*LoopInfo)->getAfterIP());
5750 ScanRedInfo->OMPScanFinish = Builder.GetInsertBlock();
5751 return Error::success();
5752 };
5753 Error Err = emitScanBasedDirectiveIR(InputLoopGen, ScanLoopGen, ScanRedInfo);
5754 if (Err)
5755 return Err;
5756 return Result;
5757}
5758
5759Value *OpenMPIRBuilder::calculateCanonicalLoopTripCount(
5760 const LocationDescription &Loc, Value *Start, Value *Stop, Value *Step,
5761 bool IsSigned, bool InclusiveStop, const Twine &Name) {
5762
5763 // Consider the following difficulties (assuming 8-bit signed integers):
5764 // * Adding \p Step to the loop counter which passes \p Stop may overflow:
5765 // DO I = 1, 100, 50
5766 /// * A \p Step of INT_MIN cannot not be normalized to a positive direction:
5767 // DO I = 100, 0, -128
5768
5769 // Start, Stop and Step must be of the same integer type.
5770 auto *IndVarTy = cast<IntegerType>(Val: Start->getType());
5771 assert(IndVarTy == Stop->getType() && "Stop type mismatch");
5772 assert(IndVarTy == Step->getType() && "Step type mismatch");
5773
5774 updateToLocation(Loc);
5775
5776 ConstantInt *Zero = ConstantInt::get(Ty: IndVarTy, V: 0);
5777 ConstantInt *One = ConstantInt::get(Ty: IndVarTy, V: 1);
5778
5779 // Like Step, but always positive.
5780 Value *Incr = Step;
5781
5782 // Distance between Start and Stop; always positive.
5783 Value *Span;
5784
5785 // Condition whether there are no iterations are executed at all, e.g. because
5786 // UB < LB.
5787 Value *ZeroCmp;
5788
5789 if (IsSigned) {
5790 // Ensure that increment is positive. If not, negate and invert LB and UB.
5791 Value *IsNeg = Builder.CreateICmpSLT(LHS: Step, RHS: Zero);
5792 Incr = Builder.CreateSelect(C: IsNeg, True: Builder.CreateNeg(V: Step), False: Step);
5793 Value *LB = Builder.CreateSelect(C: IsNeg, True: Stop, False: Start);
5794 Value *UB = Builder.CreateSelect(C: IsNeg, True: Start, False: Stop);
5795 Span = Builder.CreateSub(LHS: UB, RHS: LB, Name: "", HasNUW: false, HasNSW: true);
5796 ZeroCmp = Builder.CreateICmp(
5797 P: InclusiveStop ? CmpInst::ICMP_SLT : CmpInst::ICMP_SLE, LHS: UB, RHS: LB);
5798 } else {
5799 Span = Builder.CreateSub(LHS: Stop, RHS: Start, Name: "", HasNUW: true);
5800 ZeroCmp = Builder.CreateICmp(
5801 P: InclusiveStop ? CmpInst::ICMP_ULT : CmpInst::ICMP_ULE, LHS: Stop, RHS: Start);
5802 }
5803
5804 Value *CountIfLooping;
5805 if (InclusiveStop) {
5806 CountIfLooping = Builder.CreateAdd(LHS: Builder.CreateUDiv(LHS: Span, RHS: Incr), RHS: One);
5807 } else {
5808 // Avoid incrementing past stop since it could overflow.
5809 Value *CountIfTwo = Builder.CreateAdd(
5810 LHS: Builder.CreateUDiv(LHS: Builder.CreateSub(LHS: Span, RHS: One), RHS: Incr), RHS: One);
5811 Value *OneCmp = Builder.CreateICmp(P: CmpInst::ICMP_ULE, LHS: Span, RHS: Incr);
5812 CountIfLooping = Builder.CreateSelect(C: OneCmp, True: One, False: CountIfTwo);
5813 }
5814
5815 return Builder.CreateSelect(C: ZeroCmp, True: Zero, False: CountIfLooping,
5816 Name: "omp_" + Name + ".tripcount");
5817}
5818
5819Expected<CanonicalLoopInfo *> OpenMPIRBuilder::createCanonicalLoop(
5820 const LocationDescription &Loc, LoopBodyGenCallbackTy BodyGenCB,
5821 Value *Start, Value *Stop, Value *Step, bool IsSigned, bool InclusiveStop,
5822 InsertPointTy ComputeIP, const Twine &Name, bool InScan,
5823 ScanInfo *ScanRedInfo) {
5824 LocationDescription ComputeLoc =
5825 ComputeIP.isSet() ? LocationDescription(ComputeIP, Loc.DL) : Loc;
5826
5827 Value *TripCount = calculateCanonicalLoopTripCount(
5828 Loc: ComputeLoc, Start, Stop, Step, IsSigned, InclusiveStop, Name);
5829
5830 auto BodyGen = [=](InsertPointTy CodeGenIP, Value *IV) {
5831 Builder.restoreIP(IP: CodeGenIP);
5832 Value *Span = Builder.CreateMul(LHS: IV, RHS: Step);
5833 Value *IndVar = Builder.CreateAdd(LHS: Span, RHS: Start);
5834 if (InScan)
5835 ScanRedInfo->IV = IndVar;
5836 return BodyGenCB(Builder.saveIP(), IndVar);
5837 };
5838 LocationDescription LoopLoc =
5839 ComputeIP.isSet()
5840 ? Loc
5841 : LocationDescription(Builder.saveIP(),
5842 Builder.getCurrentDebugLocation());
5843 return createCanonicalLoop(Loc: LoopLoc, BodyGenCB: BodyGen, TripCount, Name);
5844}
5845
5846// Returns an LLVM function to call for initializing loop bounds using OpenMP
5847// static scheduling for composite `distribute parallel for` depending on
5848// `type`. Only i32 and i64 are supported by the runtime. Always interpret
5849// integers as unsigned similarly to CanonicalLoopInfo.
5850static FunctionCallee
5851getKmpcDistForStaticInitForType(Type *Ty, Module &M,
5852 OpenMPIRBuilder &OMPBuilder) {
5853 unsigned Bitwidth = Ty->getIntegerBitWidth();
5854 if (Bitwidth == 32)
5855 return OMPBuilder.getOrCreateRuntimeFunction(
5856 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dist_for_static_init_4u);
5857 if (Bitwidth == 64)
5858 return OMPBuilder.getOrCreateRuntimeFunction(
5859 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dist_for_static_init_8u);
5860 llvm_unreachable("unknown OpenMP loop iterator bitwidth");
5861}
5862
5863// Returns an LLVM function to call for initializing loop bounds using OpenMP
5864// static scheduling depending on `type`. Only i32 and i64 are supported by the
5865// runtime. Always interpret integers as unsigned similarly to
5866// CanonicalLoopInfo.
5867static FunctionCallee getKmpcForStaticInitForType(Type *Ty, Module &M,
5868 OpenMPIRBuilder &OMPBuilder) {
5869 unsigned Bitwidth = Ty->getIntegerBitWidth();
5870 if (Bitwidth == 32)
5871 return OMPBuilder.getOrCreateRuntimeFunction(
5872 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_4u);
5873 if (Bitwidth == 64)
5874 return OMPBuilder.getOrCreateRuntimeFunction(
5875 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_8u);
5876 llvm_unreachable("unknown OpenMP loop iterator bitwidth");
5877}
5878
5879OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::applyStaticWorkshareLoop(
5880 DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP,
5881 WorksharingLoopType LoopType, bool NeedsBarrier, bool HasDistSchedule,
5882 OMPScheduleType DistScheduleSchedType) {
5883 assert(CLI->isValid() && "Requires a valid canonical loop");
5884 assert(!isConflictIP(AllocaIP, CLI->getPreheaderIP()) &&
5885 "Require dedicated allocate IP");
5886
5887 // Set up the source location value for OpenMP runtime.
5888 Builder.restoreIP(IP: CLI->getPreheaderIP());
5889 Builder.SetCurrentDebugLocation(DL);
5890
5891 uint32_t SrcLocStrSize;
5892 Constant *SrcLocStr = getOrCreateSrcLocStr(DL, SrcLocStrSize);
5893 IdentFlag Flag = IdentFlag(0);
5894 switch (LoopType) {
5895 case WorksharingLoopType::ForStaticLoop:
5896 Flag = OMP_IDENT_FLAG_WORK_LOOP;
5897 break;
5898 case WorksharingLoopType::DistributeStaticLoop:
5899 Flag = OMP_IDENT_FLAG_WORK_DISTRIBUTE;
5900 break;
5901 case WorksharingLoopType::DistributeForStaticLoop:
5902 Flag = OMP_IDENT_FLAG_WORK_DISTRIBUTE | OMP_IDENT_FLAG_WORK_LOOP;
5903 break;
5904 }
5905 Value *SrcLoc = getOrCreateIdent(SrcLocStr, SrcLocStrSize, LocFlags: Flag);
5906
5907 // Declare useful OpenMP runtime functions.
5908 Value *IV = CLI->getIndVar();
5909 Type *IVTy = IV->getType();
5910 FunctionCallee StaticInit =
5911 LoopType == WorksharingLoopType::DistributeForStaticLoop
5912 ? getKmpcDistForStaticInitForType(Ty: IVTy, M, OMPBuilder&: *this)
5913 : getKmpcForStaticInitForType(Ty: IVTy, M, OMPBuilder&: *this);
5914 FunctionCallee StaticFini =
5915 getOrCreateRuntimeFunction(M, FnID: omp::OMPRTL___kmpc_for_static_fini);
5916
5917 // Allocate space for computed loop bounds as expected by the "init" function.
5918 Builder.SetInsertPoint(AllocaIP.getBlock()->getFirstNonPHIOrDbgOrAlloca());
5919
5920 Type *I32Type = Type::getInt32Ty(C&: M.getContext());
5921 Value *PLastIter = Builder.CreateAlloca(Ty: I32Type, ArraySize: nullptr, Name: "p.lastiter");
5922 Value *PLowerBound = Builder.CreateAlloca(Ty: IVTy, ArraySize: nullptr, Name: "p.lowerbound");
5923 Value *PUpperBound = Builder.CreateAlloca(Ty: IVTy, ArraySize: nullptr, Name: "p.upperbound");
5924 Value *PStride = Builder.CreateAlloca(Ty: IVTy, ArraySize: nullptr, Name: "p.stride");
5925 CLI->setLastIter(PLastIter);
5926
5927 // At the end of the preheader, prepare for calling the "init" function by
5928 // storing the current loop bounds into the allocated space. A canonical loop
5929 // always iterates from 0 to trip-count with step 1. Note that "init" expects
5930 // and produces an inclusive upper bound.
5931 Builder.SetInsertPoint(CLI->getPreheader()->getTerminator());
5932 Constant *Zero = ConstantInt::get(Ty: IVTy, V: 0);
5933 Constant *One = ConstantInt::get(Ty: IVTy, V: 1);
5934 Builder.CreateStore(Val: Zero, Ptr: PLowerBound);
5935 Value *UpperBound = Builder.CreateSub(LHS: CLI->getTripCount(), RHS: One);
5936 Builder.CreateStore(Val: UpperBound, Ptr: PUpperBound);
5937 Builder.CreateStore(Val: One, Ptr: PStride);
5938
5939 Value *ThreadNum =
5940 getOrCreateThreadID(Ident: getOrCreateIdent(SrcLocStr, SrcLocStrSize));
5941
5942 OMPScheduleType SchedType =
5943 (LoopType == WorksharingLoopType::DistributeStaticLoop)
5944 ? OMPScheduleType::OrderedDistribute
5945 : OMPScheduleType::UnorderedStatic;
5946 Constant *SchedulingType =
5947 ConstantInt::get(Ty: I32Type, V: static_cast<int>(SchedType));
5948
5949 // Call the "init" function and update the trip count of the loop with the
5950 // value it produced.
5951 auto BuildInitCall = [LoopType, SrcLoc, ThreadNum, PLastIter, PLowerBound,
5952 PUpperBound, IVTy, PStride, One, Zero, StaticInit,
5953 this](Value *SchedulingType, auto &Builder) {
5954 SmallVector<Value *, 10> Args({SrcLoc, ThreadNum, SchedulingType, PLastIter,
5955 PLowerBound, PUpperBound});
5956 if (LoopType == WorksharingLoopType::DistributeForStaticLoop) {
5957 Value *PDistUpperBound =
5958 Builder.CreateAlloca(IVTy, nullptr, "p.distupperbound");
5959 Args.push_back(Elt: PDistUpperBound);
5960 }
5961 Args.append(IL: {PStride, One, Zero});
5962 createRuntimeFunctionCall(Callee: StaticInit, Args);
5963 };
5964 BuildInitCall(SchedulingType, Builder);
5965 if (HasDistSchedule &&
5966 LoopType != WorksharingLoopType::DistributeStaticLoop) {
5967 Constant *DistScheduleSchedType = ConstantInt::get(
5968 Ty: I32Type, V: static_cast<int>(omp::OMPScheduleType::OrderedDistribute));
5969 // We want to emit a second init function call for the dist_schedule clause
5970 // to the Distribute construct. This should only be done however if a
5971 // Workshare Loop is nested within a Distribute Construct
5972 BuildInitCall(DistScheduleSchedType, Builder);
5973 }
5974 Value *LowerBound = Builder.CreateLoad(Ty: IVTy, Ptr: PLowerBound);
5975 Value *InclusiveUpperBound = Builder.CreateLoad(Ty: IVTy, Ptr: PUpperBound);
5976 Value *TripCountMinusOne = Builder.CreateSub(LHS: InclusiveUpperBound, RHS: LowerBound);
5977 Value *TripCount = Builder.CreateAdd(LHS: TripCountMinusOne, RHS: One);
5978 CLI->setTripCount(TripCount);
5979
5980 // Update all uses of the induction variable except the one in the condition
5981 // block that compares it with the actual upper bound, and the increment in
5982 // the latch block.
5983
5984 CLI->mapIndVar(Updater: [&](Instruction *OldIV) -> Value * {
5985 Builder.SetInsertPoint(TheBB: CLI->getBody(),
5986 IP: CLI->getBody()->getFirstInsertionPt());
5987 Builder.SetCurrentDebugLocation(DL);
5988 return Builder.CreateAdd(LHS: OldIV, RHS: LowerBound);
5989 });
5990
5991 // In the "exit" block, call the "fini" function.
5992 Builder.SetInsertPoint(TheBB: CLI->getExit(),
5993 IP: CLI->getExit()->getTerminator()->getIterator());
5994 createRuntimeFunctionCall(Callee: StaticFini, Args: {SrcLoc, ThreadNum});
5995
5996 // Add the barrier if requested.
5997 if (NeedsBarrier) {
5998 InsertPointOrErrorTy BarrierIP =
5999 createBarrier(Loc: LocationDescription(Builder.saveIP(), DL),
6000 Kind: omp::Directive::OMPD_for, /* ForceSimpleCall */ false,
6001 /* CheckCancelFlag */ false);
6002 if (!BarrierIP)
6003 return BarrierIP.takeError();
6004 }
6005
6006 InsertPointTy AfterIP = CLI->getAfterIP();
6007 CLI->invalidate();
6008
6009 return AfterIP;
6010}
6011
6012static void addAccessGroupMetadata(BasicBlock *Block, MDNode *AccessGroup,
6013 LoopInfo &LI);
6014static void addLoopMetadata(CanonicalLoopInfo *Loop,
6015 ArrayRef<Metadata *> Properties);
6016
6017static void applyParallelAccessesMetadata(CanonicalLoopInfo *CLI,
6018 LLVMContext &Ctx, Loop *Loop,
6019 LoopInfo &LoopInfo,
6020 SmallVector<Metadata *> &LoopMDList) {
6021 SmallSet<BasicBlock *, 8> Reachable;
6022
6023 // Get the basic blocks from the loop in which memref instructions
6024 // can be found.
6025 // TODO: Generalize getting all blocks inside a CanonicalizeLoopInfo,
6026 // preferably without running any passes.
6027 for (BasicBlock *Block : Loop->getBlocks()) {
6028 if (Block == CLI->getCond() || Block == CLI->getHeader())
6029 continue;
6030 Reachable.insert(Ptr: Block);
6031 }
6032
6033 // Add access group metadata to memory-access instructions.
6034 MDNode *AccessGroup = MDNode::getDistinct(Context&: Ctx, MDs: {});
6035 for (BasicBlock *BB : Reachable)
6036 addAccessGroupMetadata(Block: BB, AccessGroup, LI&: LoopInfo);
6037 // TODO: If the loop has existing parallel access metadata, have
6038 // to combine two lists.
6039 LoopMDList.push_back(Elt: MDNode::get(
6040 Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.parallel_accesses"), AccessGroup}));
6041}
6042
6043OpenMPIRBuilder::InsertPointOrErrorTy
6044OpenMPIRBuilder::applyStaticChunkedWorkshareLoop(
6045 DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP,
6046 bool NeedsBarrier, Value *ChunkSize, OMPScheduleType SchedType,
6047 Value *DistScheduleChunkSize, OMPScheduleType DistScheduleSchedType) {
6048 assert(CLI->isValid() && "Requires a valid canonical loop");
6049 assert((ChunkSize || DistScheduleChunkSize) && "Chunk size is required");
6050
6051 LLVMContext &Ctx = CLI->getFunction()->getContext();
6052 Value *IV = CLI->getIndVar();
6053 Value *OrigTripCount = CLI->getTripCount();
6054 Type *IVTy = IV->getType();
6055 assert(IVTy->getIntegerBitWidth() <= 64 &&
6056 "Max supported tripcount bitwidth is 64 bits");
6057 Type *InternalIVTy = IVTy->getIntegerBitWidth() <= 32 ? Type::getInt32Ty(C&: Ctx)
6058 : Type::getInt64Ty(C&: Ctx);
6059 Type *I32Type = Type::getInt32Ty(C&: M.getContext());
6060 Constant *Zero = ConstantInt::get(Ty: InternalIVTy, V: 0);
6061 Constant *One = ConstantInt::get(Ty: InternalIVTy, V: 1);
6062
6063 Function *F = CLI->getFunction();
6064 // Blocks must have terminators.
6065 // FIXME: Don't run analyses on incomplete/invalid IR.
6066 SmallVector<Instruction *> UIs;
6067 for (BasicBlock &BB : *F)
6068 if (!BB.hasTerminator())
6069 UIs.push_back(Elt: new UnreachableInst(F->getContext(), &BB));
6070 FunctionAnalysisManager FAM;
6071 FAM.registerPass(PassBuilder: []() { return DominatorTreeAnalysis(); });
6072 FAM.registerPass(PassBuilder: []() { return PassInstrumentationAnalysis(); });
6073 LoopAnalysis LIA;
6074 LoopInfo &&LI = LIA.run(F&: *F, AM&: FAM);
6075 for (Instruction *I : UIs)
6076 I->eraseFromParent();
6077 Loop *L = LI.getLoopFor(BB: CLI->getHeader());
6078 SmallVector<Metadata *> LoopMDList;
6079 if (ChunkSize || DistScheduleChunkSize)
6080 applyParallelAccessesMetadata(CLI, Ctx, Loop: L, LoopInfo&: LI, LoopMDList);
6081 addLoopMetadata(Loop: CLI, Properties: LoopMDList);
6082
6083 // Declare useful OpenMP runtime functions.
6084 FunctionCallee StaticInit =
6085 getKmpcForStaticInitForType(Ty: InternalIVTy, M, OMPBuilder&: *this);
6086 FunctionCallee StaticFini =
6087 getOrCreateRuntimeFunction(M, FnID: omp::OMPRTL___kmpc_for_static_fini);
6088
6089 // Allocate space for computed loop bounds as expected by the "init" function.
6090 Builder.restoreIP(IP: AllocaIP);
6091 Builder.SetCurrentDebugLocation(DL);
6092 Value *PLastIter = Builder.CreateAlloca(Ty: I32Type, ArraySize: nullptr, Name: "p.lastiter");
6093 Value *PLowerBound =
6094 Builder.CreateAlloca(Ty: InternalIVTy, ArraySize: nullptr, Name: "p.lowerbound");
6095 Value *PUpperBound =
6096 Builder.CreateAlloca(Ty: InternalIVTy, ArraySize: nullptr, Name: "p.upperbound");
6097 Value *PStride = Builder.CreateAlloca(Ty: InternalIVTy, ArraySize: nullptr, Name: "p.stride");
6098 CLI->setLastIter(PLastIter);
6099
6100 // Set up the source location value for the OpenMP runtime.
6101 Builder.restoreIP(IP: CLI->getPreheaderIP());
6102 Builder.SetCurrentDebugLocation(DL);
6103
6104 // TODO: Detect overflow in ubsan or max-out with current tripcount.
6105 Value *CastedChunkSize = Builder.CreateZExtOrTrunc(
6106 V: ChunkSize ? ChunkSize : Zero, DestTy: InternalIVTy, Name: "chunksize");
6107 Value *CastedDistScheduleChunkSize = Builder.CreateZExtOrTrunc(
6108 V: DistScheduleChunkSize ? DistScheduleChunkSize : Zero, DestTy: InternalIVTy,
6109 Name: "distschedulechunksize");
6110 Value *CastedTripCount =
6111 Builder.CreateZExt(V: OrigTripCount, DestTy: InternalIVTy, Name: "tripcount");
6112
6113 Constant *SchedulingType =
6114 ConstantInt::get(Ty: I32Type, V: static_cast<int>(SchedType));
6115 Constant *DistSchedulingType =
6116 ConstantInt::get(Ty: I32Type, V: static_cast<int>(DistScheduleSchedType));
6117 Builder.CreateStore(Val: Zero, Ptr: PLowerBound);
6118 Value *OrigUpperBound = Builder.CreateSub(LHS: CastedTripCount, RHS: One);
6119 Value *IsTripCountZero = Builder.CreateICmpEQ(LHS: CastedTripCount, RHS: Zero);
6120 Value *UpperBound =
6121 Builder.CreateSelect(C: IsTripCountZero, True: Zero, False: OrigUpperBound);
6122 Builder.CreateStore(Val: UpperBound, Ptr: PUpperBound);
6123 Builder.CreateStore(Val: One, Ptr: PStride);
6124
6125 // Call the "init" function and update the trip count of the loop with the
6126 // value it produced.
6127 uint32_t SrcLocStrSize;
6128 Constant *SrcLocStr = getOrCreateSrcLocStr(DL, SrcLocStrSize);
6129 IdentFlag Flag = OMP_IDENT_FLAG_WORK_LOOP;
6130 if (DistScheduleSchedType != OMPScheduleType::None) {
6131 Flag |= OMP_IDENT_FLAG_WORK_DISTRIBUTE;
6132 }
6133 Value *SrcLoc = getOrCreateIdent(SrcLocStr, SrcLocStrSize, LocFlags: Flag);
6134 Value *ThreadNum =
6135 getOrCreateThreadID(Ident: getOrCreateIdent(SrcLocStr, SrcLocStrSize));
6136 auto BuildInitCall = [StaticInit, SrcLoc, ThreadNum, PLastIter, PLowerBound,
6137 PUpperBound, PStride, One,
6138 this](Value *SchedulingType, Value *ChunkSize,
6139 auto &Builder) {
6140 createRuntimeFunctionCall(
6141 Callee: StaticInit, Args: {/*loc=*/SrcLoc, /*global_tid=*/ThreadNum,
6142 /*schedtype=*/SchedulingType, /*plastiter=*/PLastIter,
6143 /*plower=*/PLowerBound, /*pupper=*/PUpperBound,
6144 /*pstride=*/PStride, /*incr=*/One,
6145 /*chunk=*/ChunkSize});
6146 };
6147 BuildInitCall(SchedulingType, CastedChunkSize, Builder);
6148 if (DistScheduleSchedType != OMPScheduleType::None &&
6149 SchedType != OMPScheduleType::OrderedDistributeChunked &&
6150 SchedType != OMPScheduleType::OrderedDistribute) {
6151 // We want to emit a second init function call for the dist_schedule clause
6152 // to the Distribute construct. This should only be done however if a
6153 // Workshare Loop is nested within a Distribute Construct
6154 BuildInitCall(DistSchedulingType, CastedDistScheduleChunkSize, Builder);
6155 }
6156
6157 // Load values written by the "init" function.
6158 Value *FirstChunkStart =
6159 Builder.CreateLoad(Ty: InternalIVTy, Ptr: PLowerBound, Name: "omp_firstchunk.lb");
6160 Value *FirstChunkStop =
6161 Builder.CreateLoad(Ty: InternalIVTy, Ptr: PUpperBound, Name: "omp_firstchunk.ub");
6162 Value *FirstChunkEnd = Builder.CreateAdd(LHS: FirstChunkStop, RHS: One);
6163 Value *ChunkRange =
6164 Builder.CreateSub(LHS: FirstChunkEnd, RHS: FirstChunkStart, Name: "omp_chunk.range");
6165 Value *NextChunkStride =
6166 Builder.CreateLoad(Ty: InternalIVTy, Ptr: PStride, Name: "omp_dispatch.stride");
6167
6168 // Create outer "dispatch" loop for enumerating the chunks.
6169 BasicBlock *DispatchEnter = splitBB(Builder, CreateBranch: true);
6170 Value *DispatchCounter;
6171
6172 // It is safe to assume this didn't return an error because the callback
6173 // passed into createCanonicalLoop is the only possible error source, and it
6174 // always returns success.
6175 CanonicalLoopInfo *DispatchCLI = cantFail(ValOrErr: createCanonicalLoop(
6176 Loc: {Builder.saveIP(), DL},
6177 BodyGenCB: [&](InsertPointTy BodyIP, Value *Counter) {
6178 DispatchCounter = Counter;
6179 return Error::success();
6180 },
6181 Start: FirstChunkStart, Stop: CastedTripCount, Step: NextChunkStride,
6182 /*IsSigned=*/false, /*InclusiveStop=*/false, /*ComputeIP=*/{},
6183 Name: "dispatch"));
6184
6185 // Remember the BasicBlocks of the dispatch loop we need, then invalidate to
6186 // not have to preserve the canonical invariant.
6187 BasicBlock *DispatchBody = DispatchCLI->getBody();
6188 BasicBlock *DispatchLatch = DispatchCLI->getLatch();
6189 BasicBlock *DispatchExit = DispatchCLI->getExit();
6190 BasicBlock *DispatchAfter = DispatchCLI->getAfter();
6191 DispatchCLI->invalidate();
6192
6193 // Rewire the original loop to become the chunk loop inside the dispatch loop.
6194 redirectTo(Source: DispatchAfter, Target: CLI->getAfter(), DL);
6195 redirectTo(Source: CLI->getExit(), Target: DispatchLatch, DL);
6196 redirectTo(Source: DispatchBody, Target: DispatchEnter, DL);
6197
6198 // Prepare the prolog of the chunk loop.
6199 Builder.restoreIP(IP: CLI->getPreheaderIP());
6200 Builder.SetCurrentDebugLocation(DL);
6201
6202 // Compute the number of iterations of the chunk loop.
6203 Builder.SetInsertPoint(CLI->getPreheader()->getTerminator());
6204 Value *ChunkEnd = Builder.CreateAdd(LHS: DispatchCounter, RHS: ChunkRange);
6205 Value *IsLastChunk =
6206 Builder.CreateICmpUGE(LHS: ChunkEnd, RHS: CastedTripCount, Name: "omp_chunk.is_last");
6207 Value *CountUntilOrigTripCount =
6208 Builder.CreateSub(LHS: CastedTripCount, RHS: DispatchCounter);
6209 Value *ChunkTripCount = Builder.CreateSelect(
6210 C: IsLastChunk, True: CountUntilOrigTripCount, False: ChunkRange, Name: "omp_chunk.tripcount");
6211 Value *BackcastedChunkTC =
6212 Builder.CreateTrunc(V: ChunkTripCount, DestTy: IVTy, Name: "omp_chunk.tripcount.trunc");
6213 CLI->setTripCount(BackcastedChunkTC);
6214
6215 // Update all uses of the induction variable except the one in the condition
6216 // block that compares it with the actual upper bound, and the increment in
6217 // the latch block.
6218 Value *BackcastedDispatchCounter =
6219 Builder.CreateTrunc(V: DispatchCounter, DestTy: IVTy, Name: "omp_dispatch.iv.trunc");
6220 CLI->mapIndVar(Updater: [&](Instruction *) -> Value * {
6221 Builder.restoreIP(IP: CLI->getBodyIP());
6222 return Builder.CreateAdd(LHS: IV, RHS: BackcastedDispatchCounter);
6223 });
6224
6225 // In the "exit" block, call the "fini" function.
6226 Builder.SetInsertPoint(TheBB: DispatchExit, IP: DispatchExit->getFirstInsertionPt());
6227 createRuntimeFunctionCall(Callee: StaticFini, Args: {SrcLoc, ThreadNum});
6228
6229 // Add the barrier if requested.
6230 if (NeedsBarrier) {
6231 InsertPointOrErrorTy AfterIP =
6232 createBarrier(Loc: LocationDescription(Builder.saveIP(), DL), Kind: OMPD_for,
6233 /*ForceSimpleCall=*/false, /*CheckCancelFlag=*/false);
6234 if (!AfterIP)
6235 return AfterIP.takeError();
6236 }
6237
6238#ifndef NDEBUG
6239 // Even though we currently do not support applying additional methods to it,
6240 // the chunk loop should remain a canonical loop.
6241 CLI->assertOK();
6242#endif
6243
6244 return InsertPointTy(DispatchAfter, DispatchAfter->getFirstInsertionPt());
6245}
6246
6247// Returns an LLVM function to call for executing an OpenMP static worksharing
6248// for loop depending on `type`. Only i32 and i64 are supported by the runtime.
6249// Always interpret integers as unsigned similarly to CanonicalLoopInfo.
6250static FunctionCallee
6251getKmpcForStaticLoopForType(Type *Ty, OpenMPIRBuilder *OMPBuilder,
6252 WorksharingLoopType LoopType) {
6253 unsigned Bitwidth = Ty->getIntegerBitWidth();
6254 Module &M = OMPBuilder->M;
6255 switch (LoopType) {
6256 case WorksharingLoopType::ForStaticLoop:
6257 if (Bitwidth == 32)
6258 return OMPBuilder->getOrCreateRuntimeFunction(
6259 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_for_static_loop_4u);
6260 if (Bitwidth == 64)
6261 return OMPBuilder->getOrCreateRuntimeFunction(
6262 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_for_static_loop_8u);
6263 break;
6264 case WorksharingLoopType::DistributeStaticLoop:
6265 if (Bitwidth == 32)
6266 return OMPBuilder->getOrCreateRuntimeFunction(
6267 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_distribute_static_loop_4u);
6268 if (Bitwidth == 64)
6269 return OMPBuilder->getOrCreateRuntimeFunction(
6270 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_distribute_static_loop_8u);
6271 break;
6272 case WorksharingLoopType::DistributeForStaticLoop:
6273 if (Bitwidth == 32)
6274 return OMPBuilder->getOrCreateRuntimeFunction(
6275 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_distribute_for_static_loop_4u);
6276 if (Bitwidth == 64)
6277 return OMPBuilder->getOrCreateRuntimeFunction(
6278 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_distribute_for_static_loop_8u);
6279 break;
6280 }
6281 if (Bitwidth != 32 && Bitwidth != 64) {
6282 llvm_unreachable("Unknown OpenMP loop iterator bitwidth");
6283 }
6284 llvm_unreachable("Unknown type of OpenMP worksharing loop");
6285}
6286
6287// Inserts a call to proper OpenMP Device RTL function which handles
6288// loop worksharing.
6289static void createTargetLoopWorkshareCall(OpenMPIRBuilder *OMPBuilder,
6290 WorksharingLoopType LoopType,
6291 BasicBlock *InsertBlock, Value *Ident,
6292 Value *LoopBodyArg, Value *TripCount,
6293 Function &LoopBodyFn, bool NoLoop) {
6294 Type *TripCountTy = TripCount->getType();
6295 Module &M = OMPBuilder->M;
6296 IRBuilder<> &Builder = OMPBuilder->Builder;
6297 FunctionCallee RTLFn =
6298 getKmpcForStaticLoopForType(Ty: TripCountTy, OMPBuilder, LoopType);
6299 SmallVector<Value *, 8> RealArgs;
6300 RealArgs.push_back(Elt: Ident);
6301 RealArgs.push_back(Elt: &LoopBodyFn);
6302 RealArgs.push_back(Elt: LoopBodyArg);
6303 RealArgs.push_back(Elt: TripCount);
6304 if (LoopType == WorksharingLoopType::DistributeStaticLoop) {
6305 RealArgs.push_back(Elt: ConstantInt::get(Ty: TripCountTy, V: 0));
6306 RealArgs.push_back(Elt: ConstantInt::get(Ty: Builder.getInt8Ty(), V: 0));
6307 Builder.restoreIP(IP: {InsertBlock, std::prev(x: InsertBlock->end())});
6308 OMPBuilder->createRuntimeFunctionCall(Callee: RTLFn, Args: RealArgs);
6309 return;
6310 }
6311 FunctionCallee RTLNumThreads = OMPBuilder->getOrCreateRuntimeFunction(
6312 M, FnID: omp::RuntimeFunction::OMPRTL_omp_get_num_threads);
6313 Builder.restoreIP(IP: {InsertBlock, std::prev(x: InsertBlock->end())});
6314 Value *NumThreads = OMPBuilder->createRuntimeFunctionCall(Callee: RTLNumThreads, Args: {});
6315
6316 RealArgs.push_back(
6317 Elt: Builder.CreateZExtOrTrunc(V: NumThreads, DestTy: TripCountTy, Name: "num.threads.cast"));
6318 RealArgs.push_back(Elt: ConstantInt::get(Ty: TripCountTy, V: 0));
6319 if (LoopType == WorksharingLoopType::DistributeForStaticLoop) {
6320 RealArgs.push_back(Elt: ConstantInt::get(Ty: TripCountTy, V: 0));
6321 RealArgs.push_back(Elt: ConstantInt::get(Ty: Builder.getInt8Ty(), V: NoLoop));
6322 } else {
6323 RealArgs.push_back(Elt: ConstantInt::get(Ty: Builder.getInt8Ty(), V: 0));
6324 }
6325
6326 OMPBuilder->createRuntimeFunctionCall(Callee: RTLFn, Args: RealArgs);
6327}
6328
6329static void workshareLoopTargetCallback(
6330 OpenMPIRBuilder *OMPIRBuilder, CanonicalLoopInfo *CLI, Value *Ident,
6331 Function &OutlinedFn, const SmallVector<Instruction *, 4> &ToBeDeleted,
6332 WorksharingLoopType LoopType, bool NoLoop) {
6333 IRBuilder<> &Builder = OMPIRBuilder->Builder;
6334 BasicBlock *Preheader = CLI->getPreheader();
6335 Value *TripCount = CLI->getTripCount();
6336
6337 // After loop body outling, the loop body contains only set up
6338 // of loop body argument structure and the call to the outlined
6339 // loop body function. Firstly, we need to move setup of loop body args
6340 // into loop preheader.
6341 Preheader->splice(ToIt: std::prev(x: Preheader->end()), FromBB: CLI->getBody(),
6342 FromBeginIt: CLI->getBody()->begin(), FromEndIt: std::prev(x: CLI->getBody()->end()));
6343
6344 // The next step is to remove the whole loop. We do not it need anymore.
6345 // That's why make an unconditional branch from loop preheader to loop
6346 // exit block
6347 Builder.restoreIP(IP: {Preheader, Preheader->end()});
6348 Builder.SetCurrentDebugLocation(Preheader->getTerminator()->getDebugLoc());
6349 Preheader->getTerminator()->eraseFromParent();
6350 Builder.CreateBr(Dest: CLI->getExit());
6351
6352 // Delete dead loop blocks
6353 OpenMPIRBuilder::OutlineInfo CleanUpInfo;
6354 SmallPtrSet<BasicBlock *, 32> RegionBlockSet;
6355 SmallVector<BasicBlock *, 32> BlocksToBeRemoved;
6356 CleanUpInfo.EntryBB = CLI->getHeader();
6357 CleanUpInfo.ExitBB = CLI->getExit();
6358 CleanUpInfo.collectBlocks(BlockSet&: RegionBlockSet, BlockVector&: BlocksToBeRemoved);
6359 DeleteDeadBlocks(BBs: BlocksToBeRemoved);
6360
6361 // Find the instruction which corresponds to loop body argument structure
6362 // and remove the call to loop body function instruction.
6363 Value *LoopBodyArg;
6364 User *OutlinedFnUser = OutlinedFn.getUniqueUndroppableUser();
6365 assert(OutlinedFnUser &&
6366 "Expected unique undroppable user of outlined function");
6367 CallInst *OutlinedFnCallInstruction = dyn_cast<CallInst>(Val: OutlinedFnUser);
6368 assert(OutlinedFnCallInstruction && "Expected outlined function call");
6369 assert((OutlinedFnCallInstruction->getParent() == Preheader) &&
6370 "Expected outlined function call to be located in loop preheader");
6371 // Check in case no argument structure has been passed.
6372 if (OutlinedFnCallInstruction->arg_size() > 1)
6373 LoopBodyArg = OutlinedFnCallInstruction->getArgOperand(i: 1);
6374 else
6375 LoopBodyArg = Constant::getNullValue(Ty: Builder.getPtrTy());
6376 OutlinedFnCallInstruction->eraseFromParent();
6377
6378 createTargetLoopWorkshareCall(OMPBuilder: OMPIRBuilder, LoopType, InsertBlock: Preheader, Ident,
6379 LoopBodyArg, TripCount, LoopBodyFn&: OutlinedFn, NoLoop);
6380
6381 for (auto &ToBeDeletedItem : ToBeDeleted)
6382 ToBeDeletedItem->eraseFromParent();
6383 CLI->invalidate();
6384}
6385
6386OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::applyWorkshareLoopTarget(
6387 DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP,
6388 WorksharingLoopType LoopType, bool NoLoop) {
6389 uint32_t SrcLocStrSize;
6390 Constant *SrcLocStr = getOrCreateSrcLocStr(DL, SrcLocStrSize);
6391 IdentFlag Flag = IdentFlag(0);
6392 switch (LoopType) {
6393 case WorksharingLoopType::ForStaticLoop:
6394 Flag = OMP_IDENT_FLAG_WORK_LOOP;
6395 break;
6396 case WorksharingLoopType::DistributeStaticLoop:
6397 Flag = OMP_IDENT_FLAG_WORK_DISTRIBUTE;
6398 break;
6399 case WorksharingLoopType::DistributeForStaticLoop:
6400 Flag = OMP_IDENT_FLAG_WORK_DISTRIBUTE | OMP_IDENT_FLAG_WORK_LOOP;
6401 break;
6402 }
6403 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize, LocFlags: Flag);
6404
6405 auto OI = std::make_unique<OutlineInfo>();
6406 OI->OuterAllocBB = CLI->getPreheader();
6407 Function *OuterFn = CLI->getPreheader()->getParent();
6408
6409 // Instructions which need to be deleted at the end of code generation
6410 SmallVector<Instruction *, 4> ToBeDeleted;
6411
6412 OI->OuterAllocBB = AllocaIP.getBlock();
6413
6414 // Mark the body loop as region which needs to be extracted
6415 OI->EntryBB = CLI->getBody();
6416 OI->ExitBB = CLI->getLatch()->splitBasicBlockBefore(I: CLI->getLatch()->begin(),
6417 BBName: "omp.prelatch");
6418
6419 // Prepare loop body for extraction
6420 Builder.restoreIP(IP: {CLI->getPreheader(), CLI->getPreheader()->begin()});
6421
6422 // Insert new loop counter variable which will be used only in loop
6423 // body.
6424 AllocaInst *NewLoopCnt = Builder.CreateAlloca(Ty: CLI->getIndVarType(), ArraySize: 0, Name: "");
6425 Instruction *NewLoopCntLoad =
6426 Builder.CreateLoad(Ty: CLI->getIndVarType(), Ptr: NewLoopCnt);
6427 // New loop counter instructions are redundant in the loop preheader when
6428 // code generation for workshare loop is finshed. That's why mark them as
6429 // ready for deletion.
6430 ToBeDeleted.push_back(Elt: NewLoopCntLoad);
6431 ToBeDeleted.push_back(Elt: NewLoopCnt);
6432
6433 // Analyse loop body region. Find all input variables which are used inside
6434 // loop body region.
6435 SmallPtrSet<BasicBlock *, 32> ParallelRegionBlockSet;
6436 SmallVector<BasicBlock *, 32> Blocks;
6437 OI->collectBlocks(BlockSet&: ParallelRegionBlockSet, BlockVector&: Blocks);
6438
6439 CodeExtractorAnalysisCache CEAC(*OuterFn);
6440 CodeExtractor Extractor(Blocks,
6441 /* DominatorTree */ nullptr,
6442 /* AggregateArgs */ true,
6443 /* BlockFrequencyInfo */ nullptr,
6444 /* BranchProbabilityInfo */ nullptr,
6445 /* AssumptionCache */ nullptr,
6446 /* AllowVarArgs */ true,
6447 /* AllowAlloca */ true,
6448 /* AllocationBlock */ CLI->getPreheader(),
6449 /* DeallocationBlocks */ {},
6450 /* Suffix */ ".omp_wsloop",
6451 /* AggrArgsIn0AddrSpace */ true);
6452
6453 BasicBlock *CommonExit = nullptr;
6454 SetVector<Value *> SinkingCands, HoistingCands;
6455
6456 // Find allocas outside the loop body region which are used inside loop
6457 // body
6458 Extractor.findAllocas(CEAC, SinkCands&: SinkingCands, HoistCands&: HoistingCands, ExitBlock&: CommonExit);
6459
6460 // We need to model loop body region as the function f(cnt, loop_arg).
6461 // That's why we replace loop induction variable by the new counter
6462 // which will be one of loop body function argument
6463 SmallVector<User *> Users(CLI->getIndVar()->user_begin(),
6464 CLI->getIndVar()->user_end());
6465 for (auto Use : Users) {
6466 if (Instruction *Inst = dyn_cast<Instruction>(Val: Use)) {
6467 if (ParallelRegionBlockSet.count(Ptr: Inst->getParent())) {
6468 Inst->replaceUsesOfWith(From: CLI->getIndVar(), To: NewLoopCntLoad);
6469 }
6470 }
6471 }
6472 // Make sure that loop counter variable is not merged into loop body
6473 // function argument structure and it is passed as separate variable
6474 OI->ExcludeArgsFromAggregate.push_back(Elt: NewLoopCntLoad);
6475
6476 // PostOutline CB is invoked when loop body function is outlined and
6477 // loop body is replaced by call to outlined function. We need to add
6478 // call to OpenMP device rtl inside loop preheader. OpenMP device rtl
6479 // function will handle loop control logic.
6480 //
6481 OI->PostOutlineCB = [=, ToBeDeletedVec =
6482 std::move(ToBeDeleted)](Function &OutlinedFn) {
6483 workshareLoopTargetCallback(OMPIRBuilder: this, CLI, Ident, OutlinedFn, ToBeDeleted: ToBeDeletedVec,
6484 LoopType, NoLoop);
6485 };
6486 addOutlineInfo(OI: std::move(OI));
6487 return CLI->getAfterIP();
6488}
6489
6490OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::applyWorkshareLoop(
6491 DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP,
6492 bool NeedsBarrier, omp::ScheduleKind SchedKind, Value *ChunkSize,
6493 bool HasSimdModifier, bool HasMonotonicModifier,
6494 bool HasNonmonotonicModifier, bool HasOrderedClause,
6495 WorksharingLoopType LoopType, bool NoLoop, bool HasDistSchedule,
6496 Value *DistScheduleChunkSize) {
6497 if (Config.isTargetDevice())
6498 return applyWorkshareLoopTarget(DL, CLI, AllocaIP, LoopType, NoLoop);
6499 OMPScheduleType EffectiveScheduleType = computeOpenMPScheduleType(
6500 ClauseKind: SchedKind, HasChunks: ChunkSize, HasSimdModifier, HasMonotonicModifier,
6501 HasNonmonotonicModifier, HasOrderedClause, HasDistScheduleChunks: DistScheduleChunkSize);
6502
6503 bool IsOrdered = (EffectiveScheduleType & OMPScheduleType::ModifierOrdered) ==
6504 OMPScheduleType::ModifierOrdered;
6505 OMPScheduleType DistScheduleSchedType = OMPScheduleType::None;
6506 if (HasDistSchedule) {
6507 DistScheduleSchedType = DistScheduleChunkSize
6508 ? OMPScheduleType::OrderedDistributeChunked
6509 : OMPScheduleType::OrderedDistribute;
6510 }
6511 switch (EffectiveScheduleType & ~OMPScheduleType::ModifierMask) {
6512 case OMPScheduleType::BaseStatic:
6513 case OMPScheduleType::BaseDistribute:
6514 assert((!ChunkSize || !DistScheduleChunkSize) &&
6515 "No chunk size with static-chunked schedule");
6516 if (IsOrdered && !HasDistSchedule)
6517 return applyDynamicWorkshareLoop(DL, CLI, AllocaIP, SchedType: EffectiveScheduleType,
6518 NeedsBarrier, Chunk: ChunkSize);
6519 // FIXME: Monotonicity ignored?
6520 if (DistScheduleChunkSize)
6521 return applyStaticChunkedWorkshareLoop(
6522 DL, CLI, AllocaIP, NeedsBarrier, ChunkSize, SchedType: EffectiveScheduleType,
6523 DistScheduleChunkSize, DistScheduleSchedType);
6524 return applyStaticWorkshareLoop(DL, CLI, AllocaIP, LoopType, NeedsBarrier,
6525 HasDistSchedule);
6526
6527 case OMPScheduleType::BaseStaticChunked:
6528 case OMPScheduleType::BaseDistributeChunked:
6529 if (IsOrdered && !HasDistSchedule)
6530 return applyDynamicWorkshareLoop(DL, CLI, AllocaIP, SchedType: EffectiveScheduleType,
6531 NeedsBarrier, Chunk: ChunkSize);
6532 // FIXME: Monotonicity ignored?
6533 return applyStaticChunkedWorkshareLoop(
6534 DL, CLI, AllocaIP, NeedsBarrier, ChunkSize, SchedType: EffectiveScheduleType,
6535 DistScheduleChunkSize, DistScheduleSchedType);
6536
6537 case OMPScheduleType::BaseRuntime:
6538 case OMPScheduleType::BaseAuto:
6539 case OMPScheduleType::BaseGreedy:
6540 case OMPScheduleType::BaseBalanced:
6541 case OMPScheduleType::BaseSteal:
6542 case OMPScheduleType::BaseRuntimeSimd:
6543 assert(!ChunkSize &&
6544 "schedule type does not support user-defined chunk sizes");
6545 [[fallthrough]];
6546 case OMPScheduleType::BaseGuidedSimd:
6547 case OMPScheduleType::BaseDynamicChunked:
6548 case OMPScheduleType::BaseGuidedChunked:
6549 case OMPScheduleType::BaseGuidedIterativeChunked:
6550 case OMPScheduleType::BaseGuidedAnalyticalChunked:
6551 case OMPScheduleType::BaseStaticBalancedChunked:
6552 return applyDynamicWorkshareLoop(DL, CLI, AllocaIP, SchedType: EffectiveScheduleType,
6553 NeedsBarrier, Chunk: ChunkSize);
6554
6555 default:
6556 llvm_unreachable("Unknown/unimplemented schedule kind");
6557 }
6558}
6559
6560/// Returns an LLVM function to call for initializing loop bounds using OpenMP
6561/// dynamic scheduling depending on `type`. Only i32 and i64 are supported by
6562/// the runtime. Always interpret integers as unsigned similarly to
6563/// CanonicalLoopInfo.
6564static FunctionCallee
6565getKmpcForDynamicInitForType(Type *Ty, Module &M, OpenMPIRBuilder &OMPBuilder) {
6566 unsigned Bitwidth = Ty->getIntegerBitWidth();
6567 if (Bitwidth == 32)
6568 return OMPBuilder.getOrCreateRuntimeFunction(
6569 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dispatch_init_4u);
6570 if (Bitwidth == 64)
6571 return OMPBuilder.getOrCreateRuntimeFunction(
6572 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dispatch_init_8u);
6573 llvm_unreachable("unknown OpenMP loop iterator bitwidth");
6574}
6575
6576/// Returns an LLVM function to call for updating the next loop using OpenMP
6577/// dynamic scheduling depending on `type`. Only i32 and i64 are supported by
6578/// the runtime. Always interpret integers as unsigned similarly to
6579/// CanonicalLoopInfo.
6580static FunctionCallee
6581getKmpcForDynamicNextForType(Type *Ty, Module &M, OpenMPIRBuilder &OMPBuilder) {
6582 unsigned Bitwidth = Ty->getIntegerBitWidth();
6583 if (Bitwidth == 32)
6584 return OMPBuilder.getOrCreateRuntimeFunction(
6585 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dispatch_next_4u);
6586 if (Bitwidth == 64)
6587 return OMPBuilder.getOrCreateRuntimeFunction(
6588 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dispatch_next_8u);
6589 llvm_unreachable("unknown OpenMP loop iterator bitwidth");
6590}
6591
6592/// Returns an LLVM function to call for finalizing the dynamic loop using
6593/// depending on `type`. Only i32 and i64 are supported by the runtime. Always
6594/// interpret integers as unsigned similarly to CanonicalLoopInfo.
6595static FunctionCallee
6596getKmpcForDynamicFiniForType(Type *Ty, Module &M, OpenMPIRBuilder &OMPBuilder) {
6597 unsigned Bitwidth = Ty->getIntegerBitWidth();
6598 if (Bitwidth == 32)
6599 return OMPBuilder.getOrCreateRuntimeFunction(
6600 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dispatch_fini_4u);
6601 if (Bitwidth == 64)
6602 return OMPBuilder.getOrCreateRuntimeFunction(
6603 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dispatch_fini_8u);
6604 llvm_unreachable("unknown OpenMP loop iterator bitwidth");
6605}
6606
6607OpenMPIRBuilder::InsertPointOrErrorTy
6608OpenMPIRBuilder::applyDynamicWorkshareLoop(DebugLoc DL, CanonicalLoopInfo *CLI,
6609 InsertPointTy AllocaIP,
6610 OMPScheduleType SchedType,
6611 bool NeedsBarrier, Value *Chunk) {
6612 assert(CLI->isValid() && "Requires a valid canonical loop");
6613 assert(!isConflictIP(AllocaIP, CLI->getPreheaderIP()) &&
6614 "Require dedicated allocate IP");
6615 assert(isValidWorkshareLoopScheduleType(SchedType) &&
6616 "Require valid schedule type");
6617
6618 bool Ordered = (SchedType & OMPScheduleType::ModifierOrdered) ==
6619 OMPScheduleType::ModifierOrdered;
6620
6621 // Set up the source location value for OpenMP runtime.
6622 Builder.SetCurrentDebugLocation(DL);
6623
6624 uint32_t SrcLocStrSize;
6625 Constant *SrcLocStr = getOrCreateSrcLocStr(DL, SrcLocStrSize);
6626 Value *SrcLoc =
6627 getOrCreateIdent(SrcLocStr, SrcLocStrSize, LocFlags: OMP_IDENT_FLAG_WORK_LOOP);
6628
6629 // Declare useful OpenMP runtime functions.
6630 Value *IV = CLI->getIndVar();
6631 Type *IVTy = IV->getType();
6632 FunctionCallee DynamicInit = getKmpcForDynamicInitForType(Ty: IVTy, M, OMPBuilder&: *this);
6633 FunctionCallee DynamicNext = getKmpcForDynamicNextForType(Ty: IVTy, M, OMPBuilder&: *this);
6634
6635 // Allocate space for computed loop bounds as expected by the "init" function.
6636 Builder.SetInsertPoint(AllocaIP.getBlock()->getFirstNonPHIOrDbgOrAlloca());
6637 Type *I32Type = Type::getInt32Ty(C&: M.getContext());
6638 Value *PLastIter = Builder.CreateAlloca(Ty: I32Type, ArraySize: nullptr, Name: "p.lastiter");
6639 Value *PLowerBound = Builder.CreateAlloca(Ty: IVTy, ArraySize: nullptr, Name: "p.lowerbound");
6640 Value *PUpperBound = Builder.CreateAlloca(Ty: IVTy, ArraySize: nullptr, Name: "p.upperbound");
6641 Value *PStride = Builder.CreateAlloca(Ty: IVTy, ArraySize: nullptr, Name: "p.stride");
6642 CLI->setLastIter(PLastIter);
6643
6644 // At the end of the preheader, prepare for calling the "init" function by
6645 // storing the current loop bounds into the allocated space. A canonical loop
6646 // always iterates from 0 to trip-count with step 1. Note that "init" expects
6647 // and produces an inclusive upper bound.
6648 BasicBlock *PreHeader = CLI->getPreheader();
6649 Builder.SetInsertPoint(PreHeader->getTerminator());
6650 Constant *One = ConstantInt::get(Ty: IVTy, V: 1);
6651 Builder.CreateStore(Val: One, Ptr: PLowerBound);
6652 Value *UpperBound = CLI->getTripCount();
6653 Builder.CreateStore(Val: UpperBound, Ptr: PUpperBound);
6654 Builder.CreateStore(Val: One, Ptr: PStride);
6655
6656 BasicBlock *Header = CLI->getHeader();
6657 BasicBlock *Exit = CLI->getExit();
6658 BasicBlock *Cond = CLI->getCond();
6659 BasicBlock *Latch = CLI->getLatch();
6660 InsertPointTy AfterIP = CLI->getAfterIP();
6661
6662 // The CLI will be "broken" in the code below, as the loop is no longer
6663 // a valid canonical loop.
6664
6665 if (!Chunk)
6666 Chunk = One;
6667
6668 Value *ThreadNum =
6669 getOrCreateThreadID(Ident: getOrCreateIdent(SrcLocStr, SrcLocStrSize));
6670
6671 Constant *SchedulingType =
6672 ConstantInt::get(Ty: I32Type, V: static_cast<int>(SchedType));
6673
6674 // Call the "init" function.
6675 createRuntimeFunctionCall(Callee: DynamicInit, Args: {SrcLoc, ThreadNum, SchedulingType,
6676 /* LowerBound */ One, UpperBound,
6677 /* step */ One, Chunk});
6678
6679 // An outer loop around the existing one.
6680 BasicBlock *OuterCond = BasicBlock::Create(
6681 Context&: PreHeader->getContext(), Name: Twine(PreHeader->getName()) + ".outer.cond",
6682 Parent: PreHeader->getParent());
6683 // This needs to be 32-bit always, so can't use the IVTy Zero above.
6684 Builder.SetInsertPoint(TheBB: OuterCond, IP: OuterCond->getFirstInsertionPt());
6685 Value *Res = createRuntimeFunctionCall(
6686 Callee: DynamicNext,
6687 Args: {SrcLoc, ThreadNum, PLastIter, PLowerBound, PUpperBound, PStride});
6688 Constant *Zero32 = ConstantInt::get(Ty: I32Type, V: 0);
6689 Value *MoreWork = Builder.CreateCmp(Pred: CmpInst::ICMP_NE, LHS: Res, RHS: Zero32);
6690 Value *LowerBound =
6691 Builder.CreateSub(LHS: Builder.CreateLoad(Ty: IVTy, Ptr: PLowerBound), RHS: One, Name: "lb");
6692 Builder.CreateCondBr(Cond: MoreWork, True: Header, False: Exit);
6693
6694 // Change PHI-node in loop header to use outer cond rather than preheader,
6695 // and set IV to the LowerBound.
6696 Instruction *Phi = &Header->front();
6697 auto *PI = cast<PHINode>(Val: Phi);
6698 PI->setIncomingBlock(i: 0, BB: OuterCond);
6699 PI->setIncomingValue(i: 0, V: LowerBound);
6700
6701 // Then set the pre-header to jump to the OuterCond
6702 Instruction *Term = PreHeader->getTerminator();
6703 auto *Br = cast<UncondBrInst>(Val: Term);
6704 Br->setSuccessor(OuterCond);
6705
6706 // Modify the inner condition:
6707 // * Use the UpperBound returned from the DynamicNext call.
6708 // * jump to the loop outer loop when done with one of the inner loops.
6709 Builder.SetInsertPoint(TheBB: Cond, IP: Cond->getFirstInsertionPt());
6710 UpperBound = Builder.CreateLoad(Ty: IVTy, Ptr: PUpperBound, Name: "ub");
6711 Instruction *Comp = &*Builder.GetInsertPoint();
6712 auto *CI = cast<CmpInst>(Val: Comp);
6713 CI->setOperand(i_nocapture: 1, Val_nocapture: UpperBound);
6714 // Redirect the inner exit to branch to outer condition.
6715 Instruction *Branch = &Cond->back();
6716 auto *BI = cast<CondBrInst>(Val: Branch);
6717 assert(BI->getSuccessor(1) == Exit);
6718 BI->setSuccessor(idx: 1, NewSucc: OuterCond);
6719
6720 // Call the "fini" function if "ordered" is present in wsloop directive.
6721 if (Ordered) {
6722 Builder.SetInsertPoint(&Latch->back());
6723 FunctionCallee DynamicFini = getKmpcForDynamicFiniForType(Ty: IVTy, M, OMPBuilder&: *this);
6724 createRuntimeFunctionCall(Callee: DynamicFini, Args: {SrcLoc, ThreadNum});
6725 }
6726
6727 // Add the barrier if requested.
6728 if (NeedsBarrier) {
6729 Builder.SetInsertPoint(&Exit->back());
6730 InsertPointOrErrorTy BarrierIP =
6731 createBarrier(Loc: LocationDescription(Builder.saveIP(), DL),
6732 Kind: omp::Directive::OMPD_for, /* ForceSimpleCall */ false,
6733 /* CheckCancelFlag */ false);
6734 if (!BarrierIP)
6735 return BarrierIP.takeError();
6736 }
6737
6738 CLI->invalidate();
6739 return AfterIP;
6740}
6741
6742/// Redirect all edges that branch to \p OldTarget to \p NewTarget. That is,
6743/// after this \p OldTarget will be orphaned.
6744static void redirectAllPredecessorsTo(BasicBlock *OldTarget,
6745 BasicBlock *NewTarget, DebugLoc DL) {
6746 for (BasicBlock *Pred : make_early_inc_range(Range: predecessors(BB: OldTarget)))
6747 redirectTo(Source: Pred, Target: NewTarget, DL);
6748}
6749
6750static void removeUnusedBlocksFromParent(ArrayRef<BasicBlock *> BBs) {
6751 SmallPtrSet<BasicBlock *, 8> InternalBBs(from_range, BBs);
6752 // We add a block to BBsToKeep iff we have proven it has an external use.
6753 SmallPtrSet<BasicBlock *, 8> BBsToKeep;
6754
6755 while (true) {
6756 bool Changed = false;
6757
6758 for (BasicBlock *BB : BBs) {
6759 if (BBsToKeep.contains(Ptr: BB))
6760 continue;
6761
6762 for (Use &U : BB->uses()) {
6763 auto *UseInst = dyn_cast<Instruction>(Val: U.getUser());
6764 if (!UseInst)
6765 continue;
6766 BasicBlock *UseBB = UseInst->getParent();
6767 if (!InternalBBs.contains(Ptr: UseBB) || BBsToKeep.contains(Ptr: UseBB)) {
6768 BBsToKeep.insert(Ptr: BB);
6769 Changed = true;
6770 break;
6771 }
6772 }
6773 }
6774
6775 if (!Changed)
6776 break;
6777 }
6778
6779 SmallVector<BasicBlock *> BBsToDelete = filter_to_vector(
6780 C&: BBs, Pred: [&BBsToKeep](BasicBlock *BB) { return !BBsToKeep.contains(Ptr: BB); });
6781 DeleteDeadBlocks(BBs: BBsToDelete);
6782}
6783
6784CanonicalLoopInfo *
6785OpenMPIRBuilder::collapseLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
6786 InsertPointTy ComputeIP) {
6787 assert(Loops.size() >= 1 && "At least one loop required");
6788 size_t NumLoops = Loops.size();
6789
6790 // Nothing to do if there is already just one loop.
6791 if (NumLoops == 1)
6792 return Loops.front();
6793
6794 CanonicalLoopInfo *Outermost = Loops.front();
6795 CanonicalLoopInfo *Innermost = Loops.back();
6796 BasicBlock *OrigPreheader = Outermost->getPreheader();
6797 BasicBlock *OrigAfter = Outermost->getAfter();
6798 Function *F = OrigPreheader->getParent();
6799
6800 // Loop control blocks that may become orphaned later.
6801 SmallVector<BasicBlock *, 12> OldControlBBs;
6802 OldControlBBs.reserve(N: 6 * Loops.size());
6803 for (CanonicalLoopInfo *Loop : Loops)
6804 Loop->collectControlBlocks(BBs&: OldControlBBs);
6805
6806 // Setup the IRBuilder for inserting the trip count computation.
6807 Builder.SetCurrentDebugLocation(DL);
6808 if (ComputeIP.isSet())
6809 Builder.restoreIP(IP: ComputeIP);
6810 else
6811 Builder.restoreIP(IP: Outermost->getPreheaderIP());
6812
6813 // Derive the collapsed' loop trip count.
6814 // TODO: Find common/largest indvar type.
6815 Value *CollapsedTripCount = nullptr;
6816 for (CanonicalLoopInfo *L : Loops) {
6817 assert(L->isValid() &&
6818 "All loops to collapse must be valid canonical loops");
6819 Value *OrigTripCount = L->getTripCount();
6820 if (!CollapsedTripCount) {
6821 CollapsedTripCount = OrigTripCount;
6822 continue;
6823 }
6824
6825 // TODO: Enable UndefinedSanitizer to diagnose an overflow here.
6826 CollapsedTripCount =
6827 Builder.CreateNUWMul(LHS: CollapsedTripCount, RHS: OrigTripCount);
6828 }
6829
6830 // Create the collapsed loop control flow.
6831 CanonicalLoopInfo *Result =
6832 createLoopSkeleton(DL, TripCount: CollapsedTripCount, F,
6833 PreInsertBefore: OrigPreheader->getNextNode(), PostInsertBefore: OrigAfter, Name: "collapsed");
6834
6835 // Build the collapsed loop body code.
6836 // Start with deriving the input loop induction variables from the collapsed
6837 // one, using a divmod scheme. To preserve the original loops' order, the
6838 // innermost loop use the least significant bits.
6839 Builder.restoreIP(IP: Result->getBodyIP());
6840
6841 Value *Leftover = Result->getIndVar();
6842 SmallVector<Value *> NewIndVars;
6843 NewIndVars.resize(N: NumLoops);
6844 for (int i = NumLoops - 1; i >= 1; --i) {
6845 Value *OrigTripCount = Loops[i]->getTripCount();
6846
6847 Value *NewIndVar = Builder.CreateURem(LHS: Leftover, RHS: OrigTripCount);
6848 NewIndVars[i] = NewIndVar;
6849
6850 Leftover = Builder.CreateUDiv(LHS: Leftover, RHS: OrigTripCount);
6851 }
6852 // Outermost loop gets all the remaining bits.
6853 NewIndVars[0] = Leftover;
6854
6855 // Construct the loop body control flow.
6856 // We progressively construct the branch structure following in direction of
6857 // the control flow, from the leading in-between code, the loop nest body, the
6858 // trailing in-between code, and rejoining the collapsed loop's latch.
6859 // ContinueBlock and ContinuePred keep track of the source(s) of next edge. If
6860 // the ContinueBlock is set, continue with that block. If ContinuePred, use
6861 // its predecessors as sources.
6862 BasicBlock *ContinueBlock = Result->getBody();
6863 BasicBlock *ContinuePred = nullptr;
6864 auto ContinueWith = [&ContinueBlock, &ContinuePred, DL](BasicBlock *Dest,
6865 BasicBlock *NextSrc) {
6866 if (ContinueBlock)
6867 redirectTo(Source: ContinueBlock, Target: Dest, DL);
6868 else
6869 redirectAllPredecessorsTo(OldTarget: ContinuePred, NewTarget: Dest, DL);
6870
6871 ContinueBlock = nullptr;
6872 ContinuePred = NextSrc;
6873 };
6874
6875 // The code before the nested loop of each level.
6876 // Because we are sinking it into the nest, it will be executed more often
6877 // that the original loop. More sophisticated schemes could keep track of what
6878 // the in-between code is and instantiate it only once per thread.
6879 for (size_t i = 0; i < NumLoops - 1; ++i)
6880 ContinueWith(Loops[i]->getBody(), Loops[i + 1]->getHeader());
6881
6882 // Connect the loop nest body.
6883 ContinueWith(Innermost->getBody(), Innermost->getLatch());
6884
6885 // The code after the nested loop at each level.
6886 for (size_t i = NumLoops - 1; i > 0; --i)
6887 ContinueWith(Loops[i]->getAfter(), Loops[i - 1]->getLatch());
6888
6889 // Connect the finished loop to the collapsed loop latch.
6890 ContinueWith(Result->getLatch(), nullptr);
6891
6892 // Replace the input loops with the new collapsed loop.
6893 redirectTo(Source: Outermost->getPreheader(), Target: Result->getPreheader(), DL);
6894 redirectTo(Source: Result->getAfter(), Target: Outermost->getAfter(), DL);
6895
6896 // Replace the input loop indvars with the derived ones.
6897 for (size_t i = 0; i < NumLoops; ++i)
6898 Loops[i]->getIndVar()->replaceAllUsesWith(V: NewIndVars[i]);
6899
6900 // Remove unused parts of the input loops.
6901 removeUnusedBlocksFromParent(BBs: OldControlBBs);
6902
6903 for (CanonicalLoopInfo *L : Loops)
6904 L->invalidate();
6905
6906#ifndef NDEBUG
6907 Result->assertOK();
6908#endif
6909 return Result;
6910}
6911
6912std::vector<CanonicalLoopInfo *>
6913OpenMPIRBuilder::tileLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
6914 ArrayRef<Value *> TileSizes) {
6915 assert(TileSizes.size() == Loops.size() &&
6916 "Must pass as many tile sizes as there are loops");
6917 int NumLoops = Loops.size();
6918 assert(NumLoops >= 1 && "At least one loop to tile required");
6919
6920 CanonicalLoopInfo *OutermostLoop = Loops.front();
6921 CanonicalLoopInfo *InnermostLoop = Loops.back();
6922 Function *F = OutermostLoop->getBody()->getParent();
6923 BasicBlock *InnerEnter = InnermostLoop->getBody();
6924 BasicBlock *InnerLatch = InnermostLoop->getLatch();
6925
6926 // Loop control blocks that may become orphaned later.
6927 SmallVector<BasicBlock *, 12> OldControlBBs;
6928 OldControlBBs.reserve(N: 6 * Loops.size());
6929 for (CanonicalLoopInfo *Loop : Loops)
6930 Loop->collectControlBlocks(BBs&: OldControlBBs);
6931
6932 // Collect original trip counts and induction variable to be accessible by
6933 // index. Also, the structure of the original loops is not preserved during
6934 // the construction of the tiled loops, so do it before we scavenge the BBs of
6935 // any original CanonicalLoopInfo.
6936 SmallVector<Value *, 4> OrigTripCounts, OrigIndVars;
6937 for (CanonicalLoopInfo *L : Loops) {
6938 assert(L->isValid() && "All input loops must be valid canonical loops");
6939 OrigTripCounts.push_back(Elt: L->getTripCount());
6940 OrigIndVars.push_back(Elt: L->getIndVar());
6941 }
6942
6943 // Collect the code between loop headers. These may contain SSA definitions
6944 // that are used in the loop nest body. To be usable with in the innermost
6945 // body, these BasicBlocks will be sunk into the loop nest body. That is,
6946 // these instructions may be executed more often than before the tiling.
6947 // TODO: It would be sufficient to only sink them into body of the
6948 // corresponding tile loop.
6949 SmallVector<std::pair<BasicBlock *, BasicBlock *>, 4> InbetweenCode;
6950 for (int i = 0; i < NumLoops - 1; ++i) {
6951 CanonicalLoopInfo *Surrounding = Loops[i];
6952 CanonicalLoopInfo *Nested = Loops[i + 1];
6953
6954 BasicBlock *EnterBB = Surrounding->getBody();
6955 BasicBlock *ExitBB = Nested->getHeader();
6956 InbetweenCode.emplace_back(Args&: EnterBB, Args&: ExitBB);
6957 }
6958
6959 // Compute the trip counts of the floor loops.
6960 Builder.SetCurrentDebugLocation(DL);
6961 Builder.restoreIP(IP: OutermostLoop->getPreheaderIP());
6962 SmallVector<Value *, 4> FloorCompleteCount, FloorCount, FloorRems;
6963 for (int i = 0; i < NumLoops; ++i) {
6964 Value *TileSize = TileSizes[i];
6965 Value *OrigTripCount = OrigTripCounts[i];
6966 Type *IVType = OrigTripCount->getType();
6967
6968 Value *FloorCompleteTripCount = Builder.CreateUDiv(LHS: OrigTripCount, RHS: TileSize);
6969 Value *FloorTripRem = Builder.CreateURem(LHS: OrigTripCount, RHS: TileSize);
6970
6971 // 0 if tripcount divides the tilesize, 1 otherwise.
6972 // 1 means we need an additional iteration for a partial tile.
6973 //
6974 // Unfortunately we cannot just use the roundup-formula
6975 // (tripcount + tilesize - 1)/tilesize
6976 // because the summation might overflow. We do not want introduce undefined
6977 // behavior when the untiled loop nest did not.
6978 Value *FloorTripOverflow =
6979 Builder.CreateICmpNE(LHS: FloorTripRem, RHS: ConstantInt::get(Ty: IVType, V: 0));
6980
6981 FloorTripOverflow = Builder.CreateZExt(V: FloorTripOverflow, DestTy: IVType);
6982 Value *FloorTripCount =
6983 Builder.CreateAdd(LHS: FloorCompleteTripCount, RHS: FloorTripOverflow,
6984 Name: "omp_floor" + Twine(i) + ".tripcount", HasNUW: true);
6985
6986 // Remember some values for later use.
6987 FloorCompleteCount.push_back(Elt: FloorCompleteTripCount);
6988 FloorCount.push_back(Elt: FloorTripCount);
6989 FloorRems.push_back(Elt: FloorTripRem);
6990 }
6991
6992 // Generate the new loop nest, from the outermost to the innermost.
6993 std::vector<CanonicalLoopInfo *> Result;
6994 Result.reserve(n: NumLoops * 2);
6995
6996 // The basic block of the surrounding loop that enters the nest generated
6997 // loop.
6998 BasicBlock *Enter = OutermostLoop->getPreheader();
6999
7000 // The basic block of the surrounding loop where the inner code should
7001 // continue.
7002 BasicBlock *Continue = OutermostLoop->getAfter();
7003
7004 // Where the next loop basic block should be inserted.
7005 BasicBlock *OutroInsertBefore = InnermostLoop->getExit();
7006
7007 auto EmbeddNewLoop =
7008 [this, DL, F, InnerEnter, &Enter, &Continue, &OutroInsertBefore](
7009 Value *TripCount, const Twine &Name) -> CanonicalLoopInfo * {
7010 CanonicalLoopInfo *EmbeddedLoop = createLoopSkeleton(
7011 DL, TripCount, F, PreInsertBefore: InnerEnter, PostInsertBefore: OutroInsertBefore, Name);
7012 redirectTo(Source: Enter, Target: EmbeddedLoop->getPreheader(), DL);
7013 redirectTo(Source: EmbeddedLoop->getAfter(), Target: Continue, DL);
7014
7015 // Setup the position where the next embedded loop connects to this loop.
7016 Enter = EmbeddedLoop->getBody();
7017 Continue = EmbeddedLoop->getLatch();
7018 OutroInsertBefore = EmbeddedLoop->getLatch();
7019 return EmbeddedLoop;
7020 };
7021
7022 auto EmbeddNewLoops = [&Result, &EmbeddNewLoop](ArrayRef<Value *> TripCounts,
7023 const Twine &NameBase) {
7024 for (auto P : enumerate(First&: TripCounts)) {
7025 CanonicalLoopInfo *EmbeddedLoop =
7026 EmbeddNewLoop(P.value(), NameBase + Twine(P.index()));
7027 Result.push_back(x: EmbeddedLoop);
7028 }
7029 };
7030
7031 EmbeddNewLoops(FloorCount, "floor");
7032
7033 // Within the innermost floor loop, emit the code that computes the tile
7034 // sizes.
7035 Builder.SetInsertPoint(Enter->getTerminator());
7036 SmallVector<Value *, 4> TileCounts;
7037 for (int i = 0; i < NumLoops; ++i) {
7038 CanonicalLoopInfo *FloorLoop = Result[i];
7039 Value *TileSize = TileSizes[i];
7040
7041 Value *FloorIsEpilogue =
7042 Builder.CreateICmpEQ(LHS: FloorLoop->getIndVar(), RHS: FloorCompleteCount[i]);
7043 Value *TileTripCount =
7044 Builder.CreateSelect(C: FloorIsEpilogue, True: FloorRems[i], False: TileSize);
7045
7046 TileCounts.push_back(Elt: TileTripCount);
7047 }
7048
7049 // Create the tile loops.
7050 EmbeddNewLoops(TileCounts, "tile");
7051
7052 // Insert the inbetween code into the body.
7053 BasicBlock *BodyEnter = Enter;
7054 BasicBlock *BodyEntered = nullptr;
7055 for (std::pair<BasicBlock *, BasicBlock *> P : InbetweenCode) {
7056 BasicBlock *EnterBB = P.first;
7057 BasicBlock *ExitBB = P.second;
7058
7059 if (BodyEnter)
7060 redirectTo(Source: BodyEnter, Target: EnterBB, DL);
7061 else
7062 redirectAllPredecessorsTo(OldTarget: BodyEntered, NewTarget: EnterBB, DL);
7063
7064 BodyEnter = nullptr;
7065 BodyEntered = ExitBB;
7066 }
7067
7068 // Append the original loop nest body into the generated loop nest body.
7069 if (BodyEnter)
7070 redirectTo(Source: BodyEnter, Target: InnerEnter, DL);
7071 else
7072 redirectAllPredecessorsTo(OldTarget: BodyEntered, NewTarget: InnerEnter, DL);
7073 redirectAllPredecessorsTo(OldTarget: InnerLatch, NewTarget: Continue, DL);
7074
7075 // Replace the original induction variable with an induction variable computed
7076 // from the tile and floor induction variables.
7077 Builder.restoreIP(IP: Result.back()->getBodyIP());
7078 for (int i = 0; i < NumLoops; ++i) {
7079 CanonicalLoopInfo *FloorLoop = Result[i];
7080 CanonicalLoopInfo *TileLoop = Result[NumLoops + i];
7081 Value *OrigIndVar = OrigIndVars[i];
7082 Value *Size = TileSizes[i];
7083
7084 Value *Scale =
7085 Builder.CreateMul(LHS: Size, RHS: FloorLoop->getIndVar(), Name: {}, /*HasNUW=*/true);
7086 Value *Shift =
7087 Builder.CreateAdd(LHS: Scale, RHS: TileLoop->getIndVar(), Name: {}, /*HasNUW=*/true);
7088 OrigIndVar->replaceAllUsesWith(V: Shift);
7089 }
7090
7091 // Remove unused parts of the original loops.
7092 removeUnusedBlocksFromParent(BBs: OldControlBBs);
7093
7094 for (CanonicalLoopInfo *L : Loops)
7095 L->invalidate();
7096
7097#ifndef NDEBUG
7098 for (CanonicalLoopInfo *GenL : Result)
7099 GenL->assertOK();
7100#endif
7101 return Result;
7102}
7103
7104/// Attach metadata \p Properties to the basic block described by \p BB. If the
7105/// basic block already has metadata, the basic block properties are appended.
7106static void addBasicBlockMetadata(BasicBlock *BB,
7107 ArrayRef<Metadata *> Properties) {
7108 // Nothing to do if no property to attach.
7109 if (Properties.empty())
7110 return;
7111
7112 LLVMContext &Ctx = BB->getContext();
7113 SmallVector<Metadata *> NewProperties;
7114 NewProperties.push_back(Elt: nullptr);
7115
7116 // If the basic block already has metadata, prepend it to the new metadata.
7117 MDNode *Existing = BB->getTerminator()->getMetadata(KindID: LLVMContext::MD_loop);
7118 if (Existing)
7119 append_range(C&: NewProperties, R: drop_begin(RangeOrContainer: Existing->operands(), N: 1));
7120
7121 append_range(C&: NewProperties, R&: Properties);
7122 MDNode *BasicBlockID = MDNode::getDistinct(Context&: Ctx, MDs: NewProperties);
7123 BasicBlockID->replaceOperandWith(I: 0, New: BasicBlockID);
7124
7125 BB->getTerminator()->setMetadata(KindID: LLVMContext::MD_loop, Node: BasicBlockID);
7126}
7127
7128/// Attach loop metadata \p Properties to the loop described by \p Loop. If the
7129/// loop already has metadata, the loop properties are appended.
7130static void addLoopMetadata(CanonicalLoopInfo *Loop,
7131 ArrayRef<Metadata *> Properties) {
7132 assert(Loop->isValid() && "Expecting a valid CanonicalLoopInfo");
7133
7134 // Attach metadata to the loop's latch
7135 BasicBlock *Latch = Loop->getLatch();
7136 assert(Latch && "A valid CanonicalLoopInfo must have a unique latch");
7137 addBasicBlockMetadata(BB: Latch, Properties);
7138}
7139
7140/// Attach llvm.access.group metadata to the memref instructions of \p Block
7141static void addAccessGroupMetadata(BasicBlock *Block, MDNode *AccessGroup,
7142 LoopInfo &LI) {
7143 for (Instruction &I : *Block) {
7144 if (I.mayReadOrWriteMemory()) {
7145 // TODO: This instruction may already have access group from
7146 // other pragmas e.g. #pragma clang loop vectorize. Append
7147 // so that the existing metadata is not overwritten.
7148 I.setMetadata(KindID: LLVMContext::MD_access_group, Node: AccessGroup);
7149 }
7150 }
7151}
7152
7153CanonicalLoopInfo *
7154OpenMPIRBuilder::fuseLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops) {
7155 CanonicalLoopInfo *firstLoop = Loops.front();
7156 CanonicalLoopInfo *lastLoop = Loops.back();
7157 Function *F = firstLoop->getPreheader()->getParent();
7158
7159 // Loop control blocks that will become orphaned later
7160 SmallVector<BasicBlock *> oldControlBBs;
7161 for (CanonicalLoopInfo *Loop : Loops)
7162 Loop->collectControlBlocks(BBs&: oldControlBBs);
7163
7164 // Collect original trip counts
7165 SmallVector<Value *> origTripCounts;
7166 for (CanonicalLoopInfo *L : Loops) {
7167 assert(L->isValid() && "All input loops must be valid canonical loops");
7168 origTripCounts.push_back(Elt: L->getTripCount());
7169 }
7170
7171 Builder.SetCurrentDebugLocation(DL);
7172
7173 // Compute max trip count.
7174 // The fused loop will be from 0 to max(origTripCounts)
7175 BasicBlock *TCBlock = BasicBlock::Create(Context&: F->getContext(), Name: "omp.fuse.comp.tc",
7176 Parent: F, InsertBefore: firstLoop->getHeader());
7177 Builder.SetInsertPoint(TCBlock);
7178 Value *fusedTripCount = nullptr;
7179 for (CanonicalLoopInfo *L : Loops) {
7180 assert(L->isValid() && "All loops to fuse must be valid canonical loops");
7181 Value *origTripCount = L->getTripCount();
7182 if (!fusedTripCount) {
7183 fusedTripCount = origTripCount;
7184 continue;
7185 }
7186 Value *condTP = Builder.CreateICmpSGT(LHS: fusedTripCount, RHS: origTripCount);
7187 fusedTripCount = Builder.CreateSelect(C: condTP, True: fusedTripCount, False: origTripCount,
7188 Name: ".omp.fuse.tc");
7189 }
7190
7191 // Generate new loop
7192 CanonicalLoopInfo *fused =
7193 createLoopSkeleton(DL, TripCount: fusedTripCount, F, PreInsertBefore: firstLoop->getBody(),
7194 PostInsertBefore: lastLoop->getLatch(), Name: "fused");
7195
7196 // Replace original loops with the fused loop
7197 // Preheader and After are not considered inside the CLI.
7198 // These are used to compute the individual TCs of the loops
7199 // so they have to be put before the resulting fused loop.
7200 // Moving them up for readability.
7201 for (size_t i = 0; i < Loops.size() - 1; ++i) {
7202 Loops[i]->getPreheader()->moveBefore(MovePos: TCBlock);
7203 Loops[i]->getAfter()->moveBefore(MovePos: TCBlock);
7204 }
7205 lastLoop->getPreheader()->moveBefore(MovePos: TCBlock);
7206
7207 for (size_t i = 0; i < Loops.size() - 1; ++i) {
7208 redirectTo(Source: Loops[i]->getPreheader(), Target: Loops[i]->getAfter(), DL);
7209 redirectTo(Source: Loops[i]->getAfter(), Target: Loops[i + 1]->getPreheader(), DL);
7210 }
7211 redirectTo(Source: lastLoop->getPreheader(), Target: TCBlock, DL);
7212 redirectTo(Source: TCBlock, Target: fused->getPreheader(), DL);
7213 redirectTo(Source: fused->getAfter(), Target: lastLoop->getAfter(), DL);
7214
7215 // Build the fused body
7216 // Create new Blocks with conditions that jump to the original loop bodies
7217 SmallVector<BasicBlock *> condBBs;
7218 SmallVector<Value *> condValues;
7219 for (size_t i = 0; i < Loops.size(); ++i) {
7220 BasicBlock *condBlock = BasicBlock::Create(
7221 Context&: F->getContext(), Name: "omp.fused.inner.cond", Parent: F, InsertBefore: Loops[i]->getBody());
7222 Builder.SetInsertPoint(condBlock);
7223 Value *condValue =
7224 Builder.CreateICmpSLT(LHS: fused->getIndVar(), RHS: origTripCounts[i]);
7225 condBBs.push_back(Elt: condBlock);
7226 condValues.push_back(Elt: condValue);
7227 }
7228 // Join the condition blocks with the bodies of the original loops
7229 redirectTo(Source: fused->getBody(), Target: condBBs[0], DL);
7230 for (size_t i = 0; i < Loops.size() - 1; ++i) {
7231 Builder.SetInsertPoint(condBBs[i]);
7232 Builder.CreateCondBr(Cond: condValues[i], True: Loops[i]->getBody(), False: condBBs[i + 1]);
7233 redirectAllPredecessorsTo(OldTarget: Loops[i]->getLatch(), NewTarget: condBBs[i + 1], DL);
7234 // Replace the IV with the fused IV
7235 Loops[i]->getIndVar()->replaceAllUsesWith(V: fused->getIndVar());
7236 }
7237 // Last body jumps to the created end body block
7238 Builder.SetInsertPoint(condBBs.back());
7239 Builder.CreateCondBr(Cond: condValues.back(), True: lastLoop->getBody(),
7240 False: fused->getLatch());
7241 redirectAllPredecessorsTo(OldTarget: lastLoop->getLatch(), NewTarget: fused->getLatch(), DL);
7242 // Replace the IV with the fused IV
7243 lastLoop->getIndVar()->replaceAllUsesWith(V: fused->getIndVar());
7244
7245 // The loop latch must have only one predecessor. Currently it is branched to
7246 // from both the last condition block and the last loop body
7247 fused->getLatch()->splitBasicBlockBefore(I: fused->getLatch()->begin(),
7248 BBName: "omp.fused.pre_latch");
7249
7250 // Remove unused parts
7251 removeUnusedBlocksFromParent(BBs: oldControlBBs);
7252
7253 // Invalidate old CLIs
7254 for (CanonicalLoopInfo *L : Loops)
7255 L->invalidate();
7256
7257#ifndef NDEBUG
7258 fused->assertOK();
7259#endif
7260 return fused;
7261}
7262
7263void OpenMPIRBuilder::unrollLoopFull(DebugLoc, CanonicalLoopInfo *Loop) {
7264 LLVMContext &Ctx = Builder.getContext();
7265 addLoopMetadata(
7266 Loop, Properties: {MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.enable")),
7267 MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.full"))});
7268}
7269
7270void OpenMPIRBuilder::unrollLoopHeuristic(DebugLoc, CanonicalLoopInfo *Loop) {
7271 LLVMContext &Ctx = Builder.getContext();
7272 addLoopMetadata(
7273 Loop, Properties: {
7274 MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.enable")),
7275 });
7276}
7277
7278void OpenMPIRBuilder::createIfVersion(CanonicalLoopInfo *CanonicalLoop,
7279 Value *IfCond, ValueToValueMapTy &VMap,
7280 LoopAnalysis &LIA, LoopInfo &LI, Loop *L,
7281 const Twine &NamePrefix) {
7282 Function *F = CanonicalLoop->getFunction();
7283
7284 // We can't do
7285 // if (cond) {
7286 // simd_loop;
7287 // } else {
7288 // non_simd_loop;
7289 // }
7290 // because then the CanonicalLoopInfo would only point to one of the loops:
7291 // leading to other constructs operating on the same loop to malfunction.
7292 // Instead generate
7293 // while (...) {
7294 // if (cond) {
7295 // simd_body;
7296 // } else {
7297 // not_simd_body;
7298 // }
7299 // }
7300 // At least for simple loops, LLVM seems able to hoist the if out of the loop
7301 // body at -O3
7302
7303 // Define where if branch should be inserted
7304 auto SplitBeforeIt = CanonicalLoop->getBody()->getFirstNonPHIIt();
7305
7306 // Create additional blocks for the if statement
7307 BasicBlock *Cond = SplitBeforeIt->getParent();
7308 llvm::LLVMContext &C = Cond->getContext();
7309 llvm::BasicBlock *ThenBlock = llvm::BasicBlock::Create(
7310 Context&: C, Name: NamePrefix + ".if.then", Parent: Cond->getParent(), InsertBefore: Cond->getNextNode());
7311 llvm::BasicBlock *ElseBlock = llvm::BasicBlock::Create(
7312 Context&: C, Name: NamePrefix + ".if.else", Parent: Cond->getParent(), InsertBefore: CanonicalLoop->getExit());
7313
7314 // Create if condition branch.
7315 Builder.SetInsertPoint(SplitBeforeIt);
7316 Instruction *BrInstr =
7317 Builder.CreateCondBr(Cond: IfCond, True: ThenBlock, /*ifFalse*/ False: ElseBlock);
7318 InsertPointTy IP{BrInstr->getParent(), ++BrInstr->getIterator()};
7319 // Then block contains branch to omp loop body which needs to be vectorized
7320 spliceBB(IP, New: ThenBlock, CreateBranch: false, DL: Builder.getCurrentDebugLocation());
7321 ThenBlock->replaceSuccessorsPhiUsesWith(Old: Cond, New: ThenBlock);
7322
7323 Builder.SetInsertPoint(ElseBlock);
7324
7325 // Clone loop for the else branch
7326 SmallVector<BasicBlock *, 8> NewBlocks;
7327
7328 SmallVector<BasicBlock *, 8> ExistingBlocks;
7329 ExistingBlocks.reserve(N: L->getNumBlocks() + 1);
7330 ExistingBlocks.push_back(Elt: ThenBlock);
7331 ExistingBlocks.append(in_start: L->block_begin(), in_end: L->block_end());
7332 // Cond is the block that has the if clause condition
7333 // LoopCond is omp_loop.cond
7334 // LoopHeader is omp_loop.header
7335 BasicBlock *LoopCond = Cond->getUniquePredecessor();
7336 BasicBlock *LoopHeader = LoopCond->getUniquePredecessor();
7337 assert(LoopCond && LoopHeader && "Invalid loop structure");
7338 for (BasicBlock *Block : ExistingBlocks) {
7339 if (Block == L->getLoopPreheader() || Block == L->getLoopLatch() ||
7340 Block == LoopHeader || Block == LoopCond || Block == Cond) {
7341 continue;
7342 }
7343 BasicBlock *NewBB = CloneBasicBlock(BB: Block, VMap, NameSuffix: "", F);
7344
7345 // fix name not to be omp.if.then
7346 if (Block == ThenBlock)
7347 NewBB->setName(NamePrefix + ".if.else");
7348
7349 NewBB->moveBefore(MovePos: CanonicalLoop->getExit());
7350 VMap[Block] = NewBB;
7351 NewBlocks.push_back(Elt: NewBB);
7352 }
7353 remapInstructionsInBlocks(Blocks: NewBlocks, VMap);
7354 Builder.CreateBr(Dest: NewBlocks.front());
7355
7356 // The loop latch must have only one predecessor. Currently it is branched to
7357 // from both the 'then' and 'else' branches.
7358 L->getLoopLatch()->splitBasicBlockBefore(I: L->getLoopLatch()->begin(),
7359 BBName: NamePrefix + ".pre_latch");
7360
7361 // Ensure that the then block is added to the loop so we add the attributes in
7362 // the next step
7363 L->addBasicBlockToLoop(NewBB: ThenBlock, LI);
7364}
7365
7366unsigned
7367OpenMPIRBuilder::getOpenMPDefaultSimdAlign(const Triple &TargetTriple,
7368 const StringMap<bool> &Features) {
7369 if (TargetTriple.isX86()) {
7370 if (Features.lookup(Key: "avx512f"))
7371 return 512;
7372 else if (Features.lookup(Key: "avx"))
7373 return 256;
7374 return 128;
7375 }
7376 if (TargetTriple.isPPC())
7377 return 128;
7378 if (TargetTriple.isWasm())
7379 return 128;
7380 return 0;
7381}
7382
7383void OpenMPIRBuilder::applySimd(CanonicalLoopInfo *CanonicalLoop,
7384 MapVector<Value *, Value *> AlignedVars,
7385 Value *IfCond, OrderKind Order,
7386 ConstantInt *Simdlen, ConstantInt *Safelen) {
7387 LLVMContext &Ctx = Builder.getContext();
7388
7389 Function *F = CanonicalLoop->getFunction();
7390
7391 // Blocks must have terminators.
7392 // FIXME: Don't run analyses on incomplete/invalid IR.
7393 SmallVector<Instruction *> UIs;
7394 for (BasicBlock &BB : *F)
7395 if (!BB.hasTerminator())
7396 UIs.push_back(Elt: new UnreachableInst(F->getContext(), &BB));
7397
7398 // TODO: We should not rely on pass manager. Currently we use pass manager
7399 // only for getting llvm::Loop which corresponds to given CanonicalLoopInfo
7400 // object. We should have a method which returns all blocks between
7401 // CanonicalLoopInfo::getHeader() and CanonicalLoopInfo::getAfter()
7402 FunctionAnalysisManager FAM;
7403 FAM.registerPass(PassBuilder: []() { return DominatorTreeAnalysis(); });
7404 FAM.registerPass(PassBuilder: []() { return LoopAnalysis(); });
7405 FAM.registerPass(PassBuilder: []() { return PassInstrumentationAnalysis(); });
7406
7407 LoopAnalysis LIA;
7408 LoopInfo &&LI = LIA.run(F&: *F, AM&: FAM);
7409
7410 for (Instruction *I : UIs)
7411 I->eraseFromParent();
7412
7413 Loop *L = LI.getLoopFor(BB: CanonicalLoop->getHeader());
7414 if (AlignedVars.size()) {
7415 InsertPointTy IP = Builder.saveIP();
7416 for (auto &AlignedItem : AlignedVars) {
7417 Value *AlignedPtr = AlignedItem.first;
7418 Value *Alignment = AlignedItem.second;
7419 Instruction *loadInst = dyn_cast<Instruction>(Val: AlignedPtr);
7420 Builder.SetInsertPoint(loadInst->getNextNode());
7421 Builder.CreateAlignmentAssumption(DL: F->getDataLayout(), PtrValue: AlignedPtr,
7422 Alignment);
7423 }
7424 Builder.restoreIP(IP);
7425 }
7426
7427 if (IfCond) {
7428 ValueToValueMapTy VMap;
7429 createIfVersion(CanonicalLoop, IfCond, VMap, LIA, LI, L, NamePrefix: "simd");
7430 }
7431
7432 SmallPtrSet<BasicBlock *, 8> Reachable;
7433
7434 // Get the basic blocks from the loop in which memref instructions
7435 // can be found.
7436 // TODO: Generalize getting all blocks inside a CanonicalizeLoopInfo,
7437 // preferably without running any passes.
7438 for (BasicBlock *Block : L->getBlocks()) {
7439 if (Block == CanonicalLoop->getCond() ||
7440 Block == CanonicalLoop->getHeader())
7441 continue;
7442 Reachable.insert(Ptr: Block);
7443 }
7444
7445 SmallVector<Metadata *> LoopMDList;
7446
7447 // In presence of finite 'safelen', it may be unsafe to mark all
7448 // the memory instructions parallel, because loop-carried
7449 // dependences of 'safelen' iterations are possible.
7450 // If clause order(concurrent) is specified then the memory instructions
7451 // are marked parallel even if 'safelen' is finite.
7452 if ((Safelen == nullptr) || (Order == OrderKind::OMP_ORDER_concurrent))
7453 applyParallelAccessesMetadata(CLI: CanonicalLoop, Ctx, Loop: L, LoopInfo&: LI, LoopMDList);
7454
7455 // FIXME: the IF clause shares a loop backedge for the SIMD and non-SIMD
7456 // versions so we can't add the loop attributes in that case.
7457 if (IfCond) {
7458 // we can still add llvm.loop.parallel_access
7459 addLoopMetadata(Loop: CanonicalLoop, Properties: LoopMDList);
7460 return;
7461 }
7462
7463 // Use the above access group metadata to create loop level
7464 // metadata, which should be distinct for each loop.
7465 ConstantAsMetadata *BoolConst =
7466 ConstantAsMetadata::get(C: ConstantInt::getTrue(Ty: Type::getInt1Ty(C&: Ctx)));
7467 LoopMDList.push_back(Elt: MDNode::get(
7468 Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.vectorize.enable"), BoolConst}));
7469
7470 if (Simdlen || Safelen) {
7471 // If both simdlen and safelen clauses are specified, the value of the
7472 // simdlen parameter must be less than or equal to the value of the safelen
7473 // parameter. Therefore, use safelen only in the absence of simdlen.
7474 ConstantInt *VectorizeWidth = Simdlen == nullptr ? Safelen : Simdlen;
7475 LoopMDList.push_back(
7476 Elt: MDNode::get(Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.vectorize.width"),
7477 ConstantAsMetadata::get(C: VectorizeWidth)}));
7478 }
7479
7480 addLoopMetadata(Loop: CanonicalLoop, Properties: LoopMDList);
7481}
7482
7483/// Create the TargetMachine object to query the backend for optimization
7484/// preferences.
7485///
7486/// Ideally, this would be passed from the front-end to the OpenMPBuilder, but
7487/// e.g. Clang does not pass it to its CodeGen layer and creates it only when
7488/// needed for the LLVM pass pipline. We use some default options to avoid
7489/// having to pass too many settings from the frontend that probably do not
7490/// matter.
7491///
7492/// Currently, TargetMachine is only used sometimes by the unrollLoopPartial
7493/// method. If we are going to use TargetMachine for more purposes, especially
7494/// those that are sensitive to TargetOptions, RelocModel and CodeModel, it
7495/// might become be worth requiring front-ends to pass on their TargetMachine,
7496/// or at least cache it between methods. Note that while fontends such as Clang
7497/// have just a single main TargetMachine per translation unit, "target-cpu" and
7498/// "target-features" that determine the TargetMachine are per-function and can
7499/// be overrided using __attribute__((target("OPTIONS"))).
7500static std::unique_ptr<TargetMachine>
7501createTargetMachine(Function *F, CodeGenOptLevel OptLevel) {
7502 Module *M = F->getParent();
7503
7504 StringRef CPU = F->getFnAttribute(Kind: "target-cpu").getValueAsString();
7505 StringRef Features = F->getFnAttribute(Kind: "target-features").getValueAsString();
7506 const llvm::Triple &Triple = M->getTargetTriple();
7507
7508 std::string Error;
7509 const llvm::Target *TheTarget = TargetRegistry::lookupTarget(TheTriple: Triple, Error);
7510 if (!TheTarget)
7511 return {};
7512
7513 llvm::TargetOptions Options;
7514 return std::unique_ptr<TargetMachine>(TheTarget->createTargetMachine(
7515 TT: Triple, CPU, Features, Options, /*RelocModel=*/RM: std::nullopt,
7516 /*CodeModel=*/CM: std::nullopt, OL: OptLevel));
7517}
7518
7519/// Heuristically determine the best-performant unroll factor for \p CLI. This
7520/// depends on the target processor. We are re-using the same heuristics as the
7521/// LoopUnrollPass.
7522static int32_t computeHeuristicUnrollFactor(CanonicalLoopInfo *CLI) {
7523 Function *F = CLI->getFunction();
7524
7525 // Assume the user requests the most aggressive unrolling, even if the rest of
7526 // the code is optimized using a lower setting.
7527 CodeGenOptLevel OptLevel = CodeGenOptLevel::Aggressive;
7528 std::unique_ptr<TargetMachine> TM = createTargetMachine(F, OptLevel);
7529
7530 // Blocks must have terminators.
7531 // FIXME: Don't run analyses on incomplete/invalid IR.
7532 SmallVector<Instruction *> UIs;
7533 for (BasicBlock &BB : *F)
7534 if (!BB.hasTerminator())
7535 UIs.push_back(Elt: new UnreachableInst(F->getContext(), &BB));
7536
7537 FunctionAnalysisManager FAM;
7538 FAM.registerPass(PassBuilder: []() { return TargetLibraryAnalysis(); });
7539 FAM.registerPass(PassBuilder: []() { return AssumptionAnalysis(); });
7540 FAM.registerPass(PassBuilder: []() { return DominatorTreeAnalysis(); });
7541 FAM.registerPass(PassBuilder: []() { return LoopAnalysis(); });
7542 FAM.registerPass(PassBuilder: []() { return ScalarEvolutionAnalysis(); });
7543 FAM.registerPass(PassBuilder: []() { return PassInstrumentationAnalysis(); });
7544 TargetIRAnalysis TIRA;
7545 if (TM)
7546 TIRA = TargetIRAnalysis(
7547 [&](const Function &F) { return TM->getTargetTransformInfo(F); });
7548 FAM.registerPass(PassBuilder: [&]() { return TIRA; });
7549
7550 TargetIRAnalysis::Result &&TTI = TIRA.run(F: *F, FAM);
7551 ScalarEvolutionAnalysis SEA;
7552 ScalarEvolution &&SE = SEA.run(F&: *F, AM&: FAM);
7553 DominatorTreeAnalysis DTA;
7554 DominatorTree &&DT = DTA.run(F&: *F, FAM);
7555 LoopAnalysis LIA;
7556 LoopInfo &&LI = LIA.run(F&: *F, AM&: FAM);
7557 AssumptionAnalysis ACT;
7558 AssumptionCache &&AC = ACT.run(F&: *F, FAM);
7559 OptimizationRemarkEmitter ORE{F};
7560
7561 for (Instruction *I : UIs)
7562 I->eraseFromParent();
7563
7564 Loop *L = LI.getLoopFor(BB: CLI->getHeader());
7565 assert(L && "Expecting CanonicalLoopInfo to be recognized as a loop");
7566
7567 TargetTransformInfo::UnrollingPreferences UP = gatherUnrollingPreferences(
7568 L, SE, TTI,
7569 /*BlockFrequencyInfo=*/BFI: nullptr,
7570 /*ProfileSummaryInfo=*/PSI: nullptr, ORE, OptLevel: static_cast<int>(OptLevel),
7571 /*UserThreshold=*/std::nullopt,
7572 /*UserAllowPartial=*/true,
7573 /*UserAllowRuntime=*/UserRuntime: true,
7574 /*UserUpperBound=*/std::nullopt,
7575 /*UserFullUnrollMaxCount=*/std::nullopt);
7576
7577 UP.Force = true;
7578
7579 // Account for additional optimizations taking place before the LoopUnrollPass
7580 // would unroll the loop.
7581 UP.Threshold *= UnrollThresholdFactor;
7582 UP.PartialThreshold *= UnrollThresholdFactor;
7583
7584 // Use normal unroll factors even if the rest of the code is optimized for
7585 // size.
7586 UP.OptSizeThreshold = UP.Threshold;
7587 UP.PartialOptSizeThreshold = UP.PartialThreshold;
7588
7589 LLVM_DEBUG(dbgs() << "Unroll heuristic thresholds:\n"
7590 << " Threshold=" << UP.Threshold << "\n"
7591 << " PartialThreshold=" << UP.PartialThreshold << "\n"
7592 << " OptSizeThreshold=" << UP.OptSizeThreshold << "\n"
7593 << " PartialOptSizeThreshold="
7594 << UP.PartialOptSizeThreshold << "\n");
7595
7596 // Disable peeling.
7597 TargetTransformInfo::PeelingPreferences PP =
7598 gatherPeelingPreferences(L, SE, TTI,
7599 /*UserAllowPeeling=*/false,
7600 /*UserAllowProfileBasedPeeling=*/false,
7601 /*UnrollingSpecficValues=*/false);
7602
7603 SmallPtrSet<const Value *, 32> EphValues;
7604 CodeMetrics::collectEphemeralValues(L, AC: &AC, EphValues);
7605
7606 // Assume that reads and writes to stack variables can be eliminated by
7607 // Mem2Reg, SROA or LICM. That is, don't count them towards the loop body's
7608 // size.
7609 for (BasicBlock *BB : L->blocks()) {
7610 for (Instruction &I : *BB) {
7611 Value *Ptr;
7612 if (auto *Load = dyn_cast<LoadInst>(Val: &I)) {
7613 Ptr = Load->getPointerOperand();
7614 } else if (auto *Store = dyn_cast<StoreInst>(Val: &I)) {
7615 Ptr = Store->getPointerOperand();
7616 } else
7617 continue;
7618
7619 Ptr = Ptr->stripPointerCasts();
7620
7621 if (auto *Alloca = dyn_cast<AllocaInst>(Val: Ptr)) {
7622 if (Alloca->getParent() == &F->getEntryBlock())
7623 EphValues.insert(Ptr: &I);
7624 }
7625 }
7626 }
7627
7628 UnrollCostEstimator UCE(L, TTI, EphValues, UP.BEInsns);
7629
7630 // Loop is not unrollable if the loop contains certain instructions.
7631 if (!UCE.canUnroll()) {
7632 LLVM_DEBUG(dbgs() << "Loop not considered unrollable\n");
7633 return 1;
7634 }
7635
7636 LLVM_DEBUG(dbgs() << "Estimated loop size is " << UCE.getRolledLoopSize()
7637 << "\n");
7638
7639 // TODO: Determine trip count of \p CLI if constant, computeUnrollCount might
7640 // be able to use it.
7641 int TripCount = 0;
7642 int MaxTripCount = 0;
7643 bool MaxOrZero = false;
7644 unsigned TripMultiple = 0;
7645
7646 unsigned Factor =
7647 computeUnrollCount(L, TTI, DT, LI: &LI, AC: &AC, SE, EphValues, ORE: &ORE, TripCount,
7648 MaxTripCount, MaxOrZero, TripMultiple, UCE, UP, PP);
7649 LLVM_DEBUG(dbgs() << "Suggesting unroll factor of " << Factor << "\n");
7650
7651 // This function returns 1 to signal to not unroll a loop.
7652 if (Factor == 0)
7653 return 1;
7654 return Factor;
7655}
7656
7657void OpenMPIRBuilder::unrollLoopPartial(DebugLoc DL, CanonicalLoopInfo *Loop,
7658 int32_t Factor,
7659 CanonicalLoopInfo **UnrolledCLI) {
7660 assert(Factor >= 0 && "Unroll factor must not be negative");
7661
7662 Function *F = Loop->getFunction();
7663 LLVMContext &Ctx = F->getContext();
7664
7665 // If the unrolled loop is not used for another loop-associated directive, it
7666 // is sufficient to add metadata for the LoopUnrollPass.
7667 if (!UnrolledCLI) {
7668 SmallVector<Metadata *, 2> LoopMetadata;
7669 LoopMetadata.push_back(
7670 Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.enable")));
7671
7672 if (Factor >= 1) {
7673 ConstantAsMetadata *FactorConst = ConstantAsMetadata::get(
7674 C: ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: APInt(32, Factor)));
7675 LoopMetadata.push_back(Elt: MDNode::get(
7676 Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.count"), FactorConst}));
7677 }
7678
7679 addLoopMetadata(Loop, Properties: LoopMetadata);
7680 return;
7681 }
7682
7683 // Heuristically determine the unroll factor.
7684 if (Factor == 0)
7685 Factor = computeHeuristicUnrollFactor(CLI: Loop);
7686
7687 // No change required with unroll factor 1.
7688 if (Factor == 1) {
7689 *UnrolledCLI = Loop;
7690 return;
7691 }
7692
7693 assert(Factor >= 2 &&
7694 "unrolling only makes sense with a factor of 2 or larger");
7695
7696 Type *IndVarTy = Loop->getIndVarType();
7697
7698 // Apply partial unrolling by tiling the loop by the unroll-factor, then fully
7699 // unroll the inner loop.
7700 Value *FactorVal =
7701 ConstantInt::get(Ty: IndVarTy, V: APInt(IndVarTy->getIntegerBitWidth(), Factor,
7702 /*isSigned=*/false));
7703 std::vector<CanonicalLoopInfo *> LoopNest =
7704 tileLoops(DL, Loops: {Loop}, TileSizes: {FactorVal});
7705 assert(LoopNest.size() == 2 && "Expect 2 loops after tiling");
7706 *UnrolledCLI = LoopNest[0];
7707 CanonicalLoopInfo *InnerLoop = LoopNest[1];
7708
7709 // LoopUnrollPass can only fully unroll loops with constant trip count.
7710 // Unroll by the unroll factor with a fallback epilog for the remainder
7711 // iterations if necessary.
7712 ConstantAsMetadata *FactorConst = ConstantAsMetadata::get(
7713 C: ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: APInt(32, Factor)));
7714 addLoopMetadata(
7715 Loop: InnerLoop,
7716 Properties: {MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.enable")),
7717 MDNode::get(
7718 Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.count"), FactorConst})});
7719
7720#ifndef NDEBUG
7721 (*UnrolledCLI)->assertOK();
7722#endif
7723}
7724
7725OpenMPIRBuilder::InsertPointTy
7726OpenMPIRBuilder::createCopyPrivate(const LocationDescription &Loc,
7727 llvm::Value *BufSize, llvm::Value *CpyBuf,
7728 llvm::Value *CpyFn, llvm::Value *DidIt) {
7729 if (!updateToLocation(Loc))
7730 return Loc.IP;
7731
7732 uint32_t SrcLocStrSize;
7733 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
7734 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
7735 Value *ThreadId = getOrCreateThreadID(Ident);
7736
7737 llvm::Value *DidItLD = Builder.CreateLoad(Ty: Builder.getInt32Ty(), Ptr: DidIt);
7738
7739 Value *Args[] = {Ident, ThreadId, BufSize, CpyBuf, CpyFn, DidItLD};
7740
7741 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_copyprivate);
7742 createRuntimeFunctionCall(Callee: Fn, Args);
7743
7744 return Builder.saveIP();
7745}
7746
7747OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createSingle(
7748 const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,
7749 FinalizeCallbackTy FiniCB, bool IsNowait, ArrayRef<llvm::Value *> CPVars,
7750 ArrayRef<llvm::Function *> CPFuncs) {
7751
7752 if (!updateToLocation(Loc))
7753 return Loc.IP;
7754
7755 // If needed allocate and initialize `DidIt` with 0.
7756 // DidIt: flag variable: 1=single thread; 0=not single thread.
7757 llvm::Value *DidIt = nullptr;
7758 if (!CPVars.empty()) {
7759 DidIt = Builder.CreateAlloca(Ty: llvm::Type::getInt32Ty(C&: Builder.getContext()));
7760 Builder.CreateStore(Val: Builder.getInt32(C: 0), Ptr: DidIt);
7761 }
7762
7763 Directive OMPD = Directive::OMPD_single;
7764 uint32_t SrcLocStrSize;
7765 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
7766 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
7767 Value *ThreadId = getOrCreateThreadID(Ident);
7768 Value *Args[] = {Ident, ThreadId};
7769
7770 Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_single);
7771 Instruction *EntryCall = createRuntimeFunctionCall(Callee: EntryRTLFn, Args);
7772
7773 Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_end_single);
7774 Instruction *ExitCall = createRuntimeFunctionCall(Callee: ExitRTLFn, Args);
7775
7776 auto FiniCBWrapper = [&](InsertPointTy IP) -> Error {
7777 if (Error Err = FiniCB(IP))
7778 return Err;
7779
7780 // The thread that executes the single region must set `DidIt` to 1.
7781 // This is used by __kmpc_copyprivate, to know if the caller is the
7782 // single thread or not.
7783 if (DidIt)
7784 Builder.CreateStore(Val: Builder.getInt32(C: 1), Ptr: DidIt);
7785
7786 return Error::success();
7787 };
7788
7789 // generates the following:
7790 // if (__kmpc_single()) {
7791 // .... single region ...
7792 // __kmpc_end_single
7793 // }
7794 // __kmpc_copyprivate
7795 // __kmpc_barrier
7796
7797 InsertPointOrErrorTy AfterIP =
7798 EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB: FiniCBWrapper,
7799 /*Conditional*/ true,
7800 /*hasFinalize*/ HasFinalize: true);
7801 if (!AfterIP)
7802 return AfterIP.takeError();
7803
7804 if (DidIt) {
7805 for (size_t I = 0, E = CPVars.size(); I < E; ++I)
7806 // NOTE BufSize is currently unused, so just pass 0.
7807 createCopyPrivate(Loc: LocationDescription(Builder.saveIP(), Loc.DL),
7808 /*BufSize=*/ConstantInt::get(Ty: Int64, V: 0), CpyBuf: CPVars[I],
7809 CpyFn: CPFuncs[I], DidIt);
7810 // NOTE __kmpc_copyprivate already inserts a barrier
7811 } else if (!IsNowait) {
7812 InsertPointOrErrorTy AfterIP =
7813 createBarrier(Loc: LocationDescription(Builder.saveIP(), Loc.DL),
7814 Kind: omp::Directive::OMPD_unknown, /* ForceSimpleCall */ false,
7815 /* CheckCancelFlag */ false);
7816 if (!AfterIP)
7817 return AfterIP.takeError();
7818 }
7819 return Builder.saveIP();
7820}
7821
7822OpenMPIRBuilder::InsertPointOrErrorTy
7823OpenMPIRBuilder::createScope(const LocationDescription &Loc,
7824 BodyGenCallbackTy BodyGenCB,
7825 FinalizeCallbackTy FiniCB, bool IsNowait) {
7826
7827 if (!updateToLocation(Loc))
7828 return Loc.IP;
7829
7830 // All threads execute the scope body — no conditional entry.
7831 InsertPointOrErrorTy AfterIP = EmitOMPInlinedRegion(
7832 OMPD: Directive::OMPD_scope, /*EntryCall=*/nullptr, /*ExitCall=*/nullptr,
7833 BodyGenCB, FiniCB, /*Conditional=*/false, /*HasFinalize=*/true,
7834 /*IsCancellable=*/false);
7835 if (!AfterIP)
7836 return AfterIP.takeError();
7837
7838 Builder.restoreIP(IP: *AfterIP);
7839 if (!IsNowait) {
7840 AfterIP = createBarrier(Loc: LocationDescription(Builder.saveIP(), Loc.DL),
7841 Kind: omp::Directive::OMPD_unknown,
7842 /*ForceSimpleCall=*/false,
7843 /*CheckCancelFlag=*/false);
7844 if (!AfterIP)
7845 return AfterIP.takeError();
7846 }
7847 return Builder.saveIP();
7848}
7849
7850OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createCritical(
7851 const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,
7852 FinalizeCallbackTy FiniCB, StringRef CriticalName, Value *HintInst) {
7853
7854 if (!updateToLocation(Loc))
7855 return Loc.IP;
7856
7857 Directive OMPD = Directive::OMPD_critical;
7858 uint32_t SrcLocStrSize;
7859 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
7860 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
7861 Value *ThreadId = getOrCreateThreadID(Ident);
7862 Value *LockVar = getOMPCriticalRegionLock(CriticalName);
7863 Value *Args[] = {Ident, ThreadId, LockVar};
7864
7865 SmallVector<llvm::Value *, 4> EnterArgs(std::begin(arr&: Args), std::end(arr&: Args));
7866 Function *RTFn = nullptr;
7867 if (HintInst) {
7868 // Add Hint to entry Args and create call
7869 EnterArgs.push_back(Elt: HintInst);
7870 RTFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_critical_with_hint);
7871 } else {
7872 RTFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_critical);
7873 }
7874 Instruction *EntryCall = createRuntimeFunctionCall(Callee: RTFn, Args: EnterArgs);
7875
7876 Function *ExitRTLFn =
7877 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_end_critical);
7878 Instruction *ExitCall = createRuntimeFunctionCall(Callee: ExitRTLFn, Args);
7879
7880 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
7881 /*Conditional*/ false, /*hasFinalize*/ HasFinalize: true);
7882}
7883
7884OpenMPIRBuilder::InsertPointTy
7885OpenMPIRBuilder::createOrderedDepend(const LocationDescription &Loc,
7886 InsertPointTy AllocaIP, unsigned NumLoops,
7887 ArrayRef<llvm::Value *> StoreValues,
7888 const Twine &Name, bool IsDependSource) {
7889 assert(
7890 llvm::all_of(StoreValues,
7891 [](Value *SV) { return SV->getType()->isIntegerTy(64); }) &&
7892 "OpenMP runtime requires depend vec with i64 type");
7893
7894 if (!updateToLocation(Loc))
7895 return Loc.IP;
7896
7897 // Allocate space for vector and generate alloc instruction.
7898 auto *ArrI64Ty = ArrayType::get(ElementType: Int64, NumElements: NumLoops);
7899 Builder.restoreIP(IP: AllocaIP);
7900 AllocaInst *ArgsBase = Builder.CreateAlloca(Ty: ArrI64Ty, ArraySize: nullptr, Name);
7901 ArgsBase->setAlignment(Align(8));
7902 updateToLocation(Loc);
7903
7904 // Store the index value with offset in depend vector.
7905 for (unsigned I = 0; I < NumLoops; ++I) {
7906 Value *DependAddrGEPIter = Builder.CreateInBoundsGEP(
7907 Ty: ArrI64Ty, Ptr: ArgsBase, IdxList: {Builder.getInt64(C: 0), Builder.getInt64(C: I)});
7908 StoreInst *STInst = Builder.CreateStore(Val: StoreValues[I], Ptr: DependAddrGEPIter);
7909 STInst->setAlignment(Align(8));
7910 }
7911
7912 Value *DependBaseAddrGEP = Builder.CreateInBoundsGEP(
7913 Ty: ArrI64Ty, Ptr: ArgsBase, IdxList: {Builder.getInt64(C: 0), Builder.getInt64(C: 0)});
7914
7915 uint32_t SrcLocStrSize;
7916 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
7917 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
7918 Value *ThreadId = getOrCreateThreadID(Ident);
7919 Value *Args[] = {Ident, ThreadId, DependBaseAddrGEP};
7920
7921 Function *RTLFn = nullptr;
7922 if (IsDependSource)
7923 RTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_doacross_post);
7924 else
7925 RTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_doacross_wait);
7926 createRuntimeFunctionCall(Callee: RTLFn, Args);
7927
7928 return Builder.saveIP();
7929}
7930
7931OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createOrderedThreadsSimd(
7932 const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,
7933 FinalizeCallbackTy FiniCB, bool IsThreads) {
7934 if (!updateToLocation(Loc))
7935 return Loc.IP;
7936
7937 Directive OMPD = Directive::OMPD_ordered;
7938 Instruction *EntryCall = nullptr;
7939 Instruction *ExitCall = nullptr;
7940
7941 if (IsThreads) {
7942 uint32_t SrcLocStrSize;
7943 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
7944 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
7945 Value *ThreadId = getOrCreateThreadID(Ident);
7946 Value *Args[] = {Ident, ThreadId};
7947
7948 Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_ordered);
7949 EntryCall = createRuntimeFunctionCall(Callee: EntryRTLFn, Args);
7950
7951 Function *ExitRTLFn =
7952 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_end_ordered);
7953 ExitCall = createRuntimeFunctionCall(Callee: ExitRTLFn, Args);
7954 }
7955
7956 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
7957 /*Conditional*/ false, /*hasFinalize*/ HasFinalize: true);
7958}
7959
7960OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::EmitOMPInlinedRegion(
7961 Directive OMPD, Instruction *EntryCall, Instruction *ExitCall,
7962 BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB, bool Conditional,
7963 bool HasFinalize, bool IsCancellable) {
7964
7965 if (HasFinalize)
7966 FinalizationStack.push_back(Elt: {FiniCB, OMPD, IsCancellable});
7967
7968 // Create inlined region's entry and body blocks, in preparation
7969 // for conditional creation
7970 BasicBlock *EntryBB = Builder.GetInsertBlock();
7971 Instruction *SplitPos = EntryBB->getTerminatorOrNull();
7972 if (!isa_and_nonnull<UncondBrInst, CondBrInst>(Val: SplitPos))
7973 SplitPos = new UnreachableInst(Builder.getContext(), EntryBB);
7974 BasicBlock *ExitBB = EntryBB->splitBasicBlock(I: SplitPos, BBName: "omp_region.end");
7975 BasicBlock *FiniBB =
7976 EntryBB->splitBasicBlock(I: EntryBB->getTerminator(), BBName: "omp_region.finalize");
7977
7978 Builder.SetInsertPoint(EntryBB->getTerminator());
7979 emitCommonDirectiveEntry(OMPD, EntryCall, ExitBB, Conditional);
7980
7981 // generate body
7982 if (Error Err =
7983 BodyGenCB(/* AllocaIP */ InsertPointTy(),
7984 /* CodeGenIP */ Builder.saveIP(), /* DeallocBlocks */ {}))
7985 return Err;
7986
7987 // emit exit call and do any needed finalization.
7988 auto FinIP = InsertPointTy(FiniBB, FiniBB->getFirstInsertionPt());
7989 assert(FiniBB->getTerminator()->getNumSuccessors() == 1 &&
7990 FiniBB->getTerminator()->getSuccessor(0) == ExitBB &&
7991 "Unexpected control flow graph state!!");
7992 InsertPointOrErrorTy AfterIP =
7993 emitCommonDirectiveExit(OMPD, FinIP, ExitCall, HasFinalize);
7994 if (!AfterIP)
7995 return AfterIP.takeError();
7996
7997 // If we are skipping the region of a non conditional, remove the exit
7998 // block, and clear the builder's insertion point.
7999 assert(SplitPos->getParent() == ExitBB &&
8000 "Unexpected Insertion point location!");
8001 auto merged = MergeBlockIntoPredecessor(BB: ExitBB);
8002 BasicBlock *ExitPredBB = SplitPos->getParent();
8003 auto InsertBB = merged ? ExitPredBB : ExitBB;
8004 if (!isa_and_nonnull<UncondBrInst, CondBrInst>(Val: SplitPos))
8005 SplitPos->eraseFromParent();
8006 Builder.SetInsertPoint(InsertBB);
8007
8008 return Builder.saveIP();
8009}
8010
8011OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitCommonDirectiveEntry(
8012 Directive OMPD, Value *EntryCall, BasicBlock *ExitBB, bool Conditional) {
8013 // if nothing to do, Return current insertion point.
8014 if (!Conditional || !EntryCall)
8015 return Builder.saveIP();
8016
8017 BasicBlock *EntryBB = Builder.GetInsertBlock();
8018 Value *CallBool = Builder.CreateIsNotNull(Arg: EntryCall);
8019 auto *ThenBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp_region.body");
8020 auto *UI = new UnreachableInst(Builder.getContext(), ThenBB);
8021
8022 // Emit thenBB and set the Builder's insertion point there for
8023 // body generation next. Place the block after the current block.
8024 Function *CurFn = EntryBB->getParent();
8025 CurFn->insert(Position: std::next(x: EntryBB->getIterator()), BB: ThenBB);
8026
8027 // Move Entry branch to end of ThenBB, and replace with conditional
8028 // branch (If-stmt)
8029 Instruction *EntryBBTI = EntryBB->getTerminator();
8030 Builder.CreateCondBr(Cond: CallBool, True: ThenBB, False: ExitBB);
8031 EntryBBTI->removeFromParent();
8032 Builder.SetInsertPoint(UI);
8033 Builder.Insert(I: EntryBBTI);
8034 UI->eraseFromParent();
8035 Builder.SetInsertPoint(ThenBB->getTerminator());
8036
8037 // return an insertion point to ExitBB.
8038 return IRBuilder<>::InsertPoint(ExitBB, ExitBB->getFirstInsertionPt());
8039}
8040
8041OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitCommonDirectiveExit(
8042 omp::Directive OMPD, InsertPointTy FinIP, Instruction *ExitCall,
8043 bool HasFinalize) {
8044
8045 Builder.restoreIP(IP: FinIP);
8046
8047 // If there is finalization to do, emit it before the exit call
8048 if (HasFinalize) {
8049 assert(!FinalizationStack.empty() &&
8050 "Unexpected finalization stack state!");
8051
8052 FinalizationInfo Fi = FinalizationStack.pop_back_val();
8053 assert(Fi.DK == OMPD && "Unexpected Directive for Finalization call!");
8054
8055 if (Error Err = Fi.mergeFiniBB(Builder, OtherFiniBB: FinIP.getBlock()))
8056 return std::move(Err);
8057
8058 // Exit condition: insertion point is before the terminator of the new Fini
8059 // block
8060 Builder.SetInsertPoint(FinIP.getBlock()->getTerminator());
8061 }
8062
8063 if (!ExitCall)
8064 return Builder.saveIP();
8065
8066 // place the Exitcall as last instruction before Finalization block terminator
8067 ExitCall->removeFromParent();
8068 Builder.Insert(I: ExitCall);
8069
8070 return IRBuilder<>::InsertPoint(ExitCall->getParent(),
8071 ExitCall->getIterator());
8072}
8073
8074OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createCopyinClauseBlocks(
8075 InsertPointTy IP, Value *MasterAddr, Value *PrivateAddr,
8076 llvm::IntegerType *IntPtrTy, bool BranchtoEnd) {
8077 if (!IP.isSet())
8078 return IP;
8079
8080 IRBuilder<>::InsertPointGuard IPG(Builder);
8081
8082 // creates the following CFG structure
8083 // OMP_Entry : (MasterAddr != PrivateAddr)?
8084 // F T
8085 // | \
8086 // | copin.not.master
8087 // | /
8088 // v /
8089 // copyin.not.master.end
8090 // |
8091 // v
8092 // OMP.Entry.Next
8093
8094 BasicBlock *OMP_Entry = IP.getBlock();
8095 Function *CurFn = OMP_Entry->getParent();
8096 BasicBlock *CopyBegin =
8097 BasicBlock::Create(Context&: M.getContext(), Name: "copyin.not.master", Parent: CurFn);
8098 BasicBlock *CopyEnd = nullptr;
8099
8100 // If entry block is terminated, split to preserve the branch to following
8101 // basic block (i.e. OMP.Entry.Next), otherwise, leave everything as is.
8102 if (isa_and_nonnull<CondBrInst>(Val: OMP_Entry->getTerminatorOrNull())) {
8103 CopyEnd = OMP_Entry->splitBasicBlock(I: OMP_Entry->getTerminator(),
8104 BBName: "copyin.not.master.end");
8105 OMP_Entry->getTerminator()->eraseFromParent();
8106 } else {
8107 CopyEnd =
8108 BasicBlock::Create(Context&: M.getContext(), Name: "copyin.not.master.end", Parent: CurFn);
8109 }
8110
8111 Builder.SetInsertPoint(OMP_Entry);
8112 Value *MasterPtr = Builder.CreatePtrToInt(V: MasterAddr, DestTy: IntPtrTy);
8113 Value *PrivatePtr = Builder.CreatePtrToInt(V: PrivateAddr, DestTy: IntPtrTy);
8114 Value *cmp = Builder.CreateICmpNE(LHS: MasterPtr, RHS: PrivatePtr);
8115 Builder.CreateCondBr(Cond: cmp, True: CopyBegin, False: CopyEnd);
8116
8117 Builder.SetInsertPoint(CopyBegin);
8118 if (BranchtoEnd)
8119 Builder.SetInsertPoint(Builder.CreateBr(Dest: CopyEnd));
8120
8121 return Builder.saveIP();
8122}
8123
8124CallInst *OpenMPIRBuilder::createOMPAlloc(const LocationDescription &Loc,
8125 Value *Size, Value *Allocator,
8126 std::string Name) {
8127 IRBuilder<>::InsertPointGuard IPG(Builder);
8128 if (!updateToLocation(Loc))
8129 return nullptr;
8130
8131 uint32_t SrcLocStrSize;
8132 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8133 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8134 Value *ThreadId = getOrCreateThreadID(Ident);
8135 Value *Args[] = {ThreadId, Size, Allocator};
8136
8137 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_alloc);
8138
8139 return createRuntimeFunctionCall(Callee: Fn, Args, Name);
8140}
8141
8142CallInst *OpenMPIRBuilder::createOMPAlignedAlloc(const LocationDescription &Loc,
8143 Value *Align, Value *Size,
8144 Value *Allocator,
8145 std::string Name) {
8146 IRBuilder<>::InsertPointGuard IPG(Builder);
8147 if (!updateToLocation(Loc))
8148 return nullptr;
8149
8150 uint32_t SrcLocStrSize;
8151 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8152 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8153 Value *ThreadId = getOrCreateThreadID(Ident);
8154 Value *Args[] = {ThreadId, Align, Size, Allocator};
8155
8156 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_aligned_alloc);
8157
8158 return Builder.CreateCall(Callee: Fn, Args, Name);
8159}
8160
8161CallInst *OpenMPIRBuilder::createOMPFree(const LocationDescription &Loc,
8162 Value *Addr, Value *Allocator,
8163 std::string Name) {
8164 IRBuilder<>::InsertPointGuard IPG(Builder);
8165 if (!updateToLocation(Loc))
8166 return nullptr;
8167
8168 uint32_t SrcLocStrSize;
8169 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8170 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8171 Value *ThreadId = getOrCreateThreadID(Ident);
8172 Value *Args[] = {ThreadId, Addr, Allocator};
8173 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_free);
8174 return createRuntimeFunctionCall(Callee: Fn, Args, Name);
8175}
8176
8177CallInst *OpenMPIRBuilder::createOMPAllocShared(const LocationDescription &Loc,
8178 Value *Size,
8179 const Twine &Name) {
8180 IRBuilder<>::InsertPointGuard IPG(Builder);
8181 updateToLocation(Loc);
8182
8183 Value *Args[] = {Size};
8184 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_alloc_shared);
8185 CallInst *Call = Builder.CreateCall(Callee: Fn, Args, Name);
8186 Call->addRetAttr(Attr: Attribute::getWithAlignment(
8187 Context&: M.getContext(), Alignment: M.getDataLayout().getPrefTypeAlign(Ty: Int64)));
8188 return Call;
8189}
8190
8191CallInst *OpenMPIRBuilder::createOMPAllocShared(const LocationDescription &Loc,
8192 Type *VarType,
8193 const Twine &Name) {
8194 return createOMPAllocShared(
8195 Loc, Size: Builder.getInt64(C: M.getDataLayout().getTypeAllocSize(Ty: VarType)), Name);
8196}
8197
8198CallInst *OpenMPIRBuilder::createOMPFreeShared(const LocationDescription &Loc,
8199 Value *Addr, Value *Size,
8200 const Twine &Name) {
8201 IRBuilder<>::InsertPointGuard IPG(Builder);
8202 updateToLocation(Loc);
8203
8204 Value *Args[] = {Addr, Size};
8205 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_free_shared);
8206 return Builder.CreateCall(Callee: Fn, Args, Name);
8207}
8208
8209CallInst *OpenMPIRBuilder::createOMPFreeShared(const LocationDescription &Loc,
8210 Value *Addr, Type *VarType,
8211 const Twine &Name) {
8212 return createOMPFreeShared(
8213 Loc, Addr, Size: Builder.getInt64(C: M.getDataLayout().getTypeAllocSize(Ty: VarType)),
8214 Name);
8215}
8216
8217CallInst *OpenMPIRBuilder::createOMPInteropInit(
8218 const LocationDescription &Loc, Value *InteropVar,
8219 omp::OMPInteropType InteropType, Value *Device, Value *NumDependences,
8220 Value *DependenceAddress, bool HaveNowaitClause) {
8221 IRBuilder<>::InsertPointGuard IPG(Builder);
8222 updateToLocation(Loc);
8223
8224 uint32_t SrcLocStrSize;
8225 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8226 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8227 Value *ThreadId = getOrCreateThreadID(Ident);
8228 if (Device == nullptr)
8229 Device = Constant::getAllOnesValue(Ty: Int32);
8230 else if (Device->getType() != Int32)
8231 Device = Builder.CreateIntCast(V: Device, DestTy: Int32, /*isSigned=*/true);
8232 Constant *InteropTypeVal = ConstantInt::get(Ty: Int32, V: (int)InteropType);
8233 if (NumDependences == nullptr) {
8234 NumDependences = ConstantInt::get(Ty: Int32, V: 0);
8235 PointerType *PointerTypeVar = PointerType::getUnqual(C&: M.getContext());
8236 DependenceAddress = ConstantPointerNull::get(T: PointerTypeVar);
8237 }
8238 Value *HaveNowaitClauseVal = ConstantInt::get(Ty: Int32, V: HaveNowaitClause);
8239 Value *Args[] = {
8240 Ident, ThreadId, InteropVar, InteropTypeVal,
8241 Device, NumDependences, DependenceAddress, HaveNowaitClauseVal};
8242
8243 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___tgt_interop_init);
8244
8245 return createRuntimeFunctionCall(Callee: Fn, Args);
8246}
8247
8248CallInst *OpenMPIRBuilder::createOMPInteropDestroy(
8249 const LocationDescription &Loc, Value *InteropVar, Value *Device,
8250 Value *NumDependences, Value *DependenceAddress, bool HaveNowaitClause) {
8251 IRBuilder<>::InsertPointGuard IPG(Builder);
8252 updateToLocation(Loc);
8253
8254 uint32_t SrcLocStrSize;
8255 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8256 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8257 Value *ThreadId = getOrCreateThreadID(Ident);
8258 if (Device == nullptr)
8259 Device = Constant::getAllOnesValue(Ty: Int32);
8260 else if (Device->getType() != Int32)
8261 Device = Builder.CreateIntCast(V: Device, DestTy: Int32, /*isSigned=*/true);
8262 if (NumDependences == nullptr) {
8263 NumDependences = ConstantInt::get(Ty: Int32, V: 0);
8264 PointerType *PointerTypeVar = PointerType::getUnqual(C&: M.getContext());
8265 DependenceAddress = ConstantPointerNull::get(T: PointerTypeVar);
8266 }
8267 Value *HaveNowaitClauseVal = ConstantInt::get(Ty: Int32, V: HaveNowaitClause);
8268 Value *Args[] = {
8269 Ident, ThreadId, InteropVar, Device,
8270 NumDependences, DependenceAddress, HaveNowaitClauseVal};
8271
8272 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___tgt_interop_destroy);
8273
8274 return createRuntimeFunctionCall(Callee: Fn, Args);
8275}
8276
8277CallInst *OpenMPIRBuilder::createOMPInteropUse(const LocationDescription &Loc,
8278 Value *InteropVar, Value *Device,
8279 Value *NumDependences,
8280 Value *DependenceAddress,
8281 bool HaveNowaitClause) {
8282 IRBuilder<>::InsertPointGuard IPG(Builder);
8283 updateToLocation(Loc);
8284 uint32_t SrcLocStrSize;
8285 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8286 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8287 Value *ThreadId = getOrCreateThreadID(Ident);
8288 if (Device == nullptr)
8289 Device = Constant::getAllOnesValue(Ty: Int32);
8290 else if (Device->getType() != Int32)
8291 Device = Builder.CreateIntCast(V: Device, DestTy: Int32, /*isSigned=*/true);
8292 if (NumDependences == nullptr) {
8293 NumDependences = ConstantInt::get(Ty: Int32, V: 0);
8294 PointerType *PointerTypeVar = PointerType::getUnqual(C&: M.getContext());
8295 DependenceAddress = ConstantPointerNull::get(T: PointerTypeVar);
8296 }
8297 Value *HaveNowaitClauseVal = ConstantInt::get(Ty: Int32, V: HaveNowaitClause);
8298 Value *Args[] = {
8299 Ident, ThreadId, InteropVar, Device,
8300 NumDependences, DependenceAddress, HaveNowaitClauseVal};
8301
8302 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___tgt_interop_use);
8303
8304 return createRuntimeFunctionCall(Callee: Fn, Args);
8305}
8306
8307CallInst *OpenMPIRBuilder::createCachedThreadPrivate(
8308 const LocationDescription &Loc, llvm::Value *Pointer,
8309 llvm::ConstantInt *Size, const llvm::Twine &Name) {
8310 IRBuilder<>::InsertPointGuard IPG(Builder);
8311 updateToLocation(Loc);
8312
8313 uint32_t SrcLocStrSize;
8314 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8315 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8316 Value *ThreadId = getOrCreateThreadID(Ident);
8317 Constant *ThreadPrivateCache =
8318 getOrCreateInternalVariable(Ty: Int8PtrPtr, Name: Name.str());
8319 llvm::Value *Args[] = {Ident, ThreadId, Pointer, Size, ThreadPrivateCache};
8320
8321 Function *Fn =
8322 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_threadprivate_cached);
8323
8324 return createRuntimeFunctionCall(Callee: Fn, Args);
8325}
8326
8327OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTargetInit(
8328 const LocationDescription &Loc,
8329 const llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs &Attrs) {
8330 assert(!Attrs.MaxThreads.empty() && !Attrs.MaxTeams.empty() &&
8331 "expected num_threads and num_teams to be specified");
8332
8333 if (!updateToLocation(Loc))
8334 return Loc.IP;
8335
8336 uint32_t SrcLocStrSize;
8337 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8338 Constant *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8339 Constant *IsSPMDVal = ConstantInt::getSigned(Ty: Int8, V: Attrs.ExecFlags);
8340 Constant *UseGenericStateMachineVal = ConstantInt::getSigned(
8341 Ty: Int8, V: Attrs.ExecFlags != omp::OMP_TGT_EXEC_MODE_SPMD &&
8342 Attrs.ExecFlags != omp::OMP_TGT_EXEC_MODE_SPMD_NO_LOOP);
8343 Constant *MayUseNestedParallelismVal = ConstantInt::getSigned(Ty: Int8, V: true);
8344 Constant *DebugIndentionLevelVal = ConstantInt::getSigned(Ty: Int16, V: 0);
8345
8346 Function *DebugKernelWrapper = Builder.GetInsertBlock()->getParent();
8347 Function *Kernel = DebugKernelWrapper;
8348
8349 // We need to strip the debug prefix to get the correct kernel name.
8350 StringRef KernelName = Kernel->getName();
8351 const std::string DebugPrefix = "_debug__";
8352 if (KernelName.ends_with(Suffix: DebugPrefix)) {
8353 KernelName = KernelName.drop_back(N: DebugPrefix.length());
8354 Kernel = M.getFunction(Name: KernelName);
8355 assert(Kernel && "Expected the real kernel to exist");
8356 }
8357
8358 // Manifest the launch configuration in the metadata matching the kernel
8359 // environment.
8360 if (Attrs.MinTeams > 1 || Attrs.MaxTeams.front() > 0)
8361 writeTeamsForKernel(T, Kernel&: *Kernel, LB: Attrs.MinTeams, UB: Attrs.MaxTeams.front());
8362
8363 // If MaxThreads is not set and needs adjustment, select the maximum between
8364 // the default workgroup size and the MinThreads value.
8365 int32_t MaxThreadsVal = Attrs.MaxThreads.front();
8366 if (MaxThreadsVal < 0 && UseDefaultMaxThreads) {
8367 if (hasGridValue(T)) {
8368 MaxThreadsVal =
8369 std::max(a: int32_t(getGridValue(T, Kernel).GV_Default_WG_Size),
8370 b: Attrs.MinThreads);
8371 } else {
8372 MaxThreadsVal = Attrs.MinThreads;
8373 }
8374 }
8375
8376 if (MaxThreadsVal > 0)
8377 writeThreadBoundsForKernel(T, Kernel&: *Kernel, LB: Attrs.MinThreads, UB: MaxThreadsVal);
8378
8379 Constant *MinThreads = ConstantInt::getSigned(Ty: Int32, V: Attrs.MinThreads);
8380 Constant *MaxThreads = ConstantInt::getSigned(Ty: Int32, V: MaxThreadsVal);
8381 Constant *MinTeams = ConstantInt::getSigned(Ty: Int32, V: Attrs.MinTeams);
8382 Constant *MaxTeams = ConstantInt::getSigned(Ty: Int32, V: Attrs.MaxTeams.front());
8383 Constant *ReductionDataSize =
8384 ConstantInt::getSigned(Ty: Int32, V: Attrs.ReductionDataSize);
8385
8386 Function *Fn = getOrCreateRuntimeFunctionPtr(
8387 FnID: omp::RuntimeFunction::OMPRTL___kmpc_target_init);
8388 const DataLayout &DL = Fn->getDataLayout();
8389
8390 Twine DynamicEnvironmentName = KernelName + "_dynamic_environment";
8391 Constant *DynamicEnvironmentInitializer =
8392 ConstantStruct::get(T: DynamicEnvironment, V: {DebugIndentionLevelVal});
8393 GlobalVariable *DynamicEnvironmentGV = new GlobalVariable(
8394 M, DynamicEnvironment, /*IsConstant=*/false, GlobalValue::WeakODRLinkage,
8395 DynamicEnvironmentInitializer, DynamicEnvironmentName,
8396 /*InsertBefore=*/nullptr, GlobalValue::NotThreadLocal,
8397 DL.getDefaultGlobalsAddressSpace());
8398 DynamicEnvironmentGV->setVisibility(GlobalValue::ProtectedVisibility);
8399
8400 Constant *DynamicEnvironment =
8401 DynamicEnvironmentGV->getType() == DynamicEnvironmentPtr
8402 ? DynamicEnvironmentGV
8403 : ConstantExpr::getAddrSpaceCast(C: DynamicEnvironmentGV,
8404 Ty: DynamicEnvironmentPtr);
8405
8406 Constant *ConfigurationEnvironmentInitializer = ConstantStruct::get(
8407 T: ConfigurationEnvironment, V: {
8408 UseGenericStateMachineVal,
8409 MayUseNestedParallelismVal,
8410 IsSPMDVal,
8411 MinThreads,
8412 MaxThreads,
8413 MinTeams,
8414 MaxTeams,
8415 ReductionDataSize,
8416 });
8417 Constant *KernelEnvironmentInitializer = ConstantStruct::get(
8418 T: KernelEnvironment, V: {
8419 ConfigurationEnvironmentInitializer,
8420 Ident,
8421 DynamicEnvironment,
8422 });
8423 std::string KernelEnvironmentName =
8424 (KernelName + "_kernel_environment").str();
8425 GlobalVariable *KernelEnvironmentGV = new GlobalVariable(
8426 M, KernelEnvironment, /*IsConstant=*/true, GlobalValue::WeakODRLinkage,
8427 KernelEnvironmentInitializer, KernelEnvironmentName,
8428 /*InsertBefore=*/nullptr, GlobalValue::NotThreadLocal,
8429 DL.getDefaultGlobalsAddressSpace());
8430 KernelEnvironmentGV->setVisibility(GlobalValue::ProtectedVisibility);
8431
8432 Constant *KernelEnvironment =
8433 KernelEnvironmentGV->getType() == KernelEnvironmentPtr
8434 ? KernelEnvironmentGV
8435 : ConstantExpr::getAddrSpaceCast(C: KernelEnvironmentGV,
8436 Ty: KernelEnvironmentPtr);
8437 Value *KernelLaunchEnvironment =
8438 DebugKernelWrapper->getArg(i: DebugKernelWrapper->arg_size() - 1);
8439 Type *KernelLaunchEnvParamTy = Fn->getFunctionType()->getParamType(i: 1);
8440 KernelLaunchEnvironment =
8441 KernelLaunchEnvironment->getType() == KernelLaunchEnvParamTy
8442 ? KernelLaunchEnvironment
8443 : Builder.CreateAddrSpaceCast(V: KernelLaunchEnvironment,
8444 DestTy: KernelLaunchEnvParamTy);
8445 CallInst *ThreadKind = createRuntimeFunctionCall(
8446 Callee: Fn, Args: {KernelEnvironment, KernelLaunchEnvironment});
8447
8448 Value *ExecUserCode = Builder.CreateICmpEQ(
8449 LHS: ThreadKind, RHS: Constant::getAllOnesValue(Ty: ThreadKind->getType()),
8450 Name: "exec_user_code");
8451
8452 // ThreadKind = __kmpc_target_init(...)
8453 // if (ThreadKind == -1)
8454 // user_code
8455 // else
8456 // return;
8457
8458 auto *UI = Builder.CreateUnreachable();
8459 BasicBlock *CheckBB = UI->getParent();
8460 BasicBlock *UserCodeEntryBB = CheckBB->splitBasicBlock(I: UI, BBName: "user_code.entry");
8461
8462 BasicBlock *WorkerExitBB = BasicBlock::Create(
8463 Context&: CheckBB->getContext(), Name: "worker.exit", Parent: CheckBB->getParent());
8464 Builder.SetInsertPoint(WorkerExitBB);
8465 Builder.CreateRetVoid();
8466
8467 auto *CheckBBTI = CheckBB->getTerminator();
8468 Builder.SetInsertPoint(CheckBBTI);
8469 Builder.CreateCondBr(Cond: ExecUserCode, True: UI->getParent(), False: WorkerExitBB);
8470
8471 CheckBBTI->eraseFromParent();
8472 UI->eraseFromParent();
8473
8474 // Continue in the "user_code" block, see diagram above and in
8475 // openmp/libomptarget/deviceRTLs/common/include/target.h .
8476 return InsertPointTy(UserCodeEntryBB, UserCodeEntryBB->getFirstInsertionPt());
8477}
8478
8479void OpenMPIRBuilder::createTargetDeinit(const LocationDescription &Loc,
8480 int32_t TeamsReductionDataSize) {
8481 if (!updateToLocation(Loc))
8482 return;
8483
8484 Function *Fn = getOrCreateRuntimeFunctionPtr(
8485 FnID: omp::RuntimeFunction::OMPRTL___kmpc_target_deinit);
8486
8487 createRuntimeFunctionCall(Callee: Fn, Args: {});
8488
8489 if (!TeamsReductionDataSize)
8490 return;
8491
8492 Function *Kernel = Builder.GetInsertBlock()->getParent();
8493 // We need to strip the debug prefix to get the correct kernel name.
8494 StringRef KernelName = Kernel->getName();
8495 const std::string DebugPrefix = "_debug__";
8496 if (KernelName.ends_with(Suffix: DebugPrefix))
8497 KernelName = KernelName.drop_back(N: DebugPrefix.length());
8498 auto *KernelEnvironmentGV =
8499 M.getNamedGlobal(Name: (KernelName + "_kernel_environment").str());
8500 assert(KernelEnvironmentGV && "Expected kernel environment global\n");
8501 auto *KernelEnvironmentInitializer = KernelEnvironmentGV->getInitializer();
8502 auto *NewInitializer = ConstantFoldInsertValueInstruction(
8503 Agg: KernelEnvironmentInitializer,
8504 Val: ConstantInt::get(Ty: Int32, V: TeamsReductionDataSize), Idxs: {0, 7});
8505 KernelEnvironmentGV->setInitializer(NewInitializer);
8506}
8507
8508static void updateNVPTXAttr(Function &Kernel, StringRef Name, int32_t Value,
8509 bool Min) {
8510 if (Kernel.hasFnAttribute(Kind: Name)) {
8511 int32_t OldLimit = Kernel.getFnAttributeAsParsedInteger(Kind: Name);
8512 Value = Min ? std::min(a: OldLimit, b: Value) : std::max(a: OldLimit, b: Value);
8513 }
8514 Kernel.addFnAttr(Kind: Name, Val: llvm::utostr(X: Value));
8515}
8516
8517std::pair<int32_t, int32_t>
8518OpenMPIRBuilder::readThreadBoundsForKernel(const Triple &T, Function &Kernel) {
8519 int32_t ThreadLimit =
8520 Kernel.getFnAttributeAsParsedInteger(Kind: "omp_target_thread_limit");
8521
8522 if (T.isAMDGPU()) {
8523 const auto &Attr = Kernel.getFnAttribute(Kind: "amdgpu-flat-work-group-size");
8524 if (!Attr.isValid() || !Attr.isStringAttribute())
8525 return {0, ThreadLimit};
8526 auto [LBStr, UBStr] = Attr.getValueAsString().split(Separator: ',');
8527 int32_t LB, UB;
8528 if (!llvm::to_integer(S: UBStr, Num&: UB, Base: 10))
8529 return {0, ThreadLimit};
8530 UB = ThreadLimit ? std::min(a: ThreadLimit, b: UB) : UB;
8531 if (!llvm::to_integer(S: LBStr, Num&: LB, Base: 10))
8532 return {0, UB};
8533 return {LB, UB};
8534 }
8535
8536 if (Kernel.hasFnAttribute(Kind: NVVMAttr::MaxNTID)) {
8537 int32_t UB = Kernel.getFnAttributeAsParsedInteger(Kind: NVVMAttr::MaxNTID);
8538 return {0, ThreadLimit ? std::min(a: ThreadLimit, b: UB) : UB};
8539 }
8540 return {0, ThreadLimit};
8541}
8542
8543void OpenMPIRBuilder::writeThreadBoundsForKernel(const Triple &T,
8544 Function &Kernel, int32_t LB,
8545 int32_t UB) {
8546 Kernel.addFnAttr(Kind: "omp_target_thread_limit", Val: std::to_string(val: UB));
8547
8548 if (T.isAMDGPU()) {
8549 Kernel.addFnAttr(Kind: "amdgpu-flat-work-group-size",
8550 Val: llvm::utostr(X: LB) + "," + llvm::utostr(X: UB));
8551 return;
8552 }
8553
8554 updateNVPTXAttr(Kernel, Name: NVVMAttr::MaxNTID, Value: UB, Min: true);
8555}
8556
8557std::pair<int32_t, int32_t>
8558OpenMPIRBuilder::readTeamBoundsForKernel(const Triple &, Function &Kernel) {
8559 // TODO: Read from backend annotations if available.
8560 return {0, Kernel.getFnAttributeAsParsedInteger(Kind: "omp_target_num_teams")};
8561}
8562
8563void OpenMPIRBuilder::writeTeamsForKernel(const Triple &T, Function &Kernel,
8564 int32_t LB, int32_t UB) {
8565 if (UB > 0) {
8566 if (T.isNVPTX())
8567 Kernel.addFnAttr(Kind: NVVMAttr::MaxClusterRank, Val: llvm::utostr(X: UB));
8568 if (T.isAMDGPU())
8569 Kernel.addFnAttr(Kind: "amdgpu-max-num-workgroups", Val: llvm::utostr(X: UB) + ",1,1");
8570 }
8571
8572 Kernel.addFnAttr(Kind: "omp_target_num_teams", Val: std::to_string(val: LB));
8573}
8574
8575void OpenMPIRBuilder::setOutlinedTargetRegionFunctionAttributes(
8576 Function *OutlinedFn) {
8577 if (Config.isTargetDevice()) {
8578 OutlinedFn->setLinkage(GlobalValue::WeakODRLinkage);
8579 // TODO: Determine if DSO local can be set to true.
8580 OutlinedFn->setDSOLocal(false);
8581 OutlinedFn->setVisibility(GlobalValue::ProtectedVisibility);
8582 if (T.isAMDGCN())
8583 OutlinedFn->setCallingConv(CallingConv::AMDGPU_KERNEL);
8584 else if (T.isNVPTX())
8585 OutlinedFn->setCallingConv(CallingConv::PTX_Kernel);
8586 else if (T.isSPIRV())
8587 OutlinedFn->setCallingConv(CallingConv::SPIR_KERNEL);
8588 }
8589}
8590
8591Constant *OpenMPIRBuilder::createOutlinedFunctionID(Function *OutlinedFn,
8592 StringRef EntryFnIDName) {
8593 if (Config.isTargetDevice()) {
8594 assert(OutlinedFn && "The outlined function must exist if embedded");
8595 return OutlinedFn;
8596 }
8597
8598 return new GlobalVariable(
8599 M, Builder.getInt8Ty(), /*isConstant=*/true, GlobalValue::WeakAnyLinkage,
8600 Constant::getNullValue(Ty: Builder.getInt8Ty()), EntryFnIDName);
8601}
8602
8603Constant *OpenMPIRBuilder::createTargetRegionEntryAddr(Function *OutlinedFn,
8604 StringRef EntryFnName) {
8605 if (OutlinedFn)
8606 return OutlinedFn;
8607
8608 assert(!M.getGlobalVariable(EntryFnName, true) &&
8609 "Named kernel already exists?");
8610 return new GlobalVariable(
8611 M, Builder.getInt8Ty(), /*isConstant=*/true, GlobalValue::InternalLinkage,
8612 Constant::getNullValue(Ty: Builder.getInt8Ty()), EntryFnName);
8613}
8614
8615Error OpenMPIRBuilder::emitTargetRegionFunction(
8616 TargetRegionEntryInfo &EntryInfo,
8617 FunctionGenCallback &GenerateFunctionCallback, bool IsOffloadEntry,
8618 Function *&OutlinedFn, Constant *&OutlinedFnID) {
8619
8620 SmallString<64> EntryFnName;
8621 OffloadInfoManager.getTargetRegionEntryFnName(Name&: EntryFnName, EntryInfo);
8622
8623 if (Config.isTargetDevice() || !Config.openMPOffloadMandatory()) {
8624 Expected<Function *> CBResult = GenerateFunctionCallback(EntryFnName);
8625 if (!CBResult)
8626 return CBResult.takeError();
8627 OutlinedFn = *CBResult;
8628 } else {
8629 OutlinedFn = nullptr;
8630 }
8631
8632 // If this target outline function is not an offload entry, we don't need to
8633 // register it. This may be in the case of a false if clause, or if there are
8634 // no OpenMP targets.
8635 if (!IsOffloadEntry)
8636 return Error::success();
8637
8638 std::string EntryFnIDName =
8639 Config.isTargetDevice()
8640 ? std::string(EntryFnName)
8641 : createPlatformSpecificName(Parts: {EntryFnName, "region_id"});
8642
8643 OutlinedFnID = registerTargetRegionFunction(EntryInfo, OutlinedFunction: OutlinedFn,
8644 EntryFnName, EntryFnIDName);
8645 return Error::success();
8646}
8647
8648Constant *OpenMPIRBuilder::registerTargetRegionFunction(
8649 TargetRegionEntryInfo &EntryInfo, Function *OutlinedFn,
8650 StringRef EntryFnName, StringRef EntryFnIDName) {
8651 if (OutlinedFn)
8652 setOutlinedTargetRegionFunctionAttributes(OutlinedFn);
8653 auto OutlinedFnID = createOutlinedFunctionID(OutlinedFn, EntryFnIDName);
8654 auto EntryAddr = createTargetRegionEntryAddr(OutlinedFn, EntryFnName);
8655 OffloadInfoManager.registerTargetRegionEntryInfo(
8656 EntryInfo, Addr: EntryAddr, ID: OutlinedFnID,
8657 Flags: OffloadEntriesInfoManager::OMPTargetRegionEntryTargetRegion);
8658 return OutlinedFnID;
8659}
8660
8661OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTargetData(
8662 const LocationDescription &Loc, InsertPointTy AllocaIP,
8663 InsertPointTy CodeGenIP, ArrayRef<BasicBlock *> DeallocBlocks,
8664 Value *DeviceID, Value *IfCond, TargetDataInfo &Info,
8665 GenMapInfoCallbackTy GenMapInfoCB, CustomMapperCallbackTy CustomMapperCB,
8666 omp::RuntimeFunction *MapperFunc,
8667 function_ref<InsertPointOrErrorTy(InsertPointTy CodeGenIP,
8668 BodyGenTy BodyGenType)>
8669 BodyGenCB,
8670 function_ref<void(unsigned int, Value *)> DeviceAddrCB, Value *SrcLocInfo) {
8671 if (!updateToLocation(Loc))
8672 return InsertPointTy();
8673
8674 Builder.restoreIP(IP: CodeGenIP);
8675
8676 bool IsStandAlone = !BodyGenCB;
8677 MapInfosTy *MapInfo;
8678 // Generate the code for the opening of the data environment. Capture all the
8679 // arguments of the runtime call by reference because they are used in the
8680 // closing of the region.
8681 auto BeginThenGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
8682 ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
8683 MapInfo = &GenMapInfoCB(Builder.saveIP());
8684 if (Error Err = emitOffloadingArrays(
8685 AllocaIP, CodeGenIP: Builder.saveIP(), CombinedInfo&: *MapInfo, Info, CustomMapperCB,
8686 /*IsNonContiguous=*/true, DeviceAddrCB))
8687 return Err;
8688
8689 TargetDataRTArgs RTArgs;
8690 emitOffloadingArraysArgument(Builder, RTArgs, Info);
8691
8692 // Emit the number of elements in the offloading arrays.
8693 Value *PointerNum = Builder.getInt32(C: Info.NumberOfPtrs);
8694
8695 // Source location for the ident struct
8696 if (!SrcLocInfo) {
8697 uint32_t SrcLocStrSize;
8698 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8699 SrcLocInfo = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8700 }
8701
8702 SmallVector<llvm::Value *, 13> OffloadingArgs = {
8703 SrcLocInfo, DeviceID,
8704 PointerNum, RTArgs.BasePointersArray,
8705 RTArgs.PointersArray, RTArgs.SizesArray,
8706 RTArgs.MapTypesArray, RTArgs.MapNamesArray,
8707 RTArgs.MappersArray};
8708
8709 if (IsStandAlone) {
8710 assert(MapperFunc && "MapperFunc missing for standalone target data");
8711
8712 auto TaskBodyCB = [&](Value *, Value *,
8713 IRBuilderBase::InsertPoint) -> Error {
8714 if (Info.HasNoWait) {
8715 OffloadingArgs.append(IL: {llvm::Constant::getNullValue(Ty: Int32),
8716 llvm::Constant::getNullValue(Ty: VoidPtr),
8717 llvm::Constant::getNullValue(Ty: Int32),
8718 llvm::Constant::getNullValue(Ty: VoidPtr)});
8719 }
8720
8721 createRuntimeFunctionCall(Callee: getOrCreateRuntimeFunctionPtr(FnID: *MapperFunc),
8722 Args: OffloadingArgs);
8723
8724 if (Info.HasNoWait) {
8725 BasicBlock *OffloadContBlock =
8726 BasicBlock::Create(Context&: Builder.getContext(), Name: "omp_offload.cont");
8727 Function *CurFn = Builder.GetInsertBlock()->getParent();
8728 emitBlock(BB: OffloadContBlock, CurFn, /*IsFinished=*/true);
8729 Builder.restoreIP(IP: Builder.saveIP());
8730 }
8731 return Error::success();
8732 };
8733
8734 bool RequiresOuterTargetTask = Info.HasNoWait;
8735 if (!RequiresOuterTargetTask)
8736 cantFail(Err: TaskBodyCB(/*DeviceID=*/nullptr, /*RTLoc=*/nullptr,
8737 /*TargetTaskAllocaIP=*/{}));
8738 else
8739 cantFail(ValOrErr: emitTargetTask(TaskBodyCB, DeviceID, RTLoc: SrcLocInfo, AllocaIP,
8740 /*Dependencies=*/{}, RTArgs, HasNoWait: Info.HasNoWait));
8741 } else {
8742 Function *BeginMapperFunc = getOrCreateRuntimeFunctionPtr(
8743 FnID: omp::OMPRTL___tgt_target_data_begin_mapper);
8744
8745 createRuntimeFunctionCall(Callee: BeginMapperFunc, Args: OffloadingArgs);
8746
8747 for (auto DeviceMap : Info.DevicePtrInfoMap) {
8748 if (isa<AllocaInst>(Val: DeviceMap.second.second)) {
8749 auto *LI =
8750 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: DeviceMap.second.first);
8751 Builder.CreateStore(Val: LI, Ptr: DeviceMap.second.second);
8752 }
8753 }
8754
8755 // If device pointer privatization is required, emit the body of the
8756 // region here. It will have to be duplicated: with and without
8757 // privatization.
8758 InsertPointOrErrorTy AfterIP =
8759 BodyGenCB(Builder.saveIP(), BodyGenTy::Priv);
8760 if (!AfterIP)
8761 return AfterIP.takeError();
8762 Builder.restoreIP(IP: *AfterIP);
8763 }
8764 return Error::success();
8765 };
8766
8767 // If we need device pointer privatization, we need to emit the body of the
8768 // region with no privatization in the 'else' branch of the conditional.
8769 // Otherwise, we don't have to do anything.
8770 auto BeginElseGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
8771 ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
8772 InsertPointOrErrorTy AfterIP =
8773 BodyGenCB(Builder.saveIP(), BodyGenTy::DupNoPriv);
8774 if (!AfterIP)
8775 return AfterIP.takeError();
8776 Builder.restoreIP(IP: *AfterIP);
8777 return Error::success();
8778 };
8779
8780 // Generate code for the closing of the data region.
8781 auto EndThenGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
8782 ArrayRef<BasicBlock *> DeallocBlocks) {
8783 TargetDataRTArgs RTArgs;
8784 Info.EmitDebug = !MapInfo->Names.empty();
8785 emitOffloadingArraysArgument(Builder, RTArgs, Info, /*ForEndCall=*/true);
8786
8787 // Emit the number of elements in the offloading arrays.
8788 Value *PointerNum = Builder.getInt32(C: Info.NumberOfPtrs);
8789
8790 // Source location for the ident struct
8791 if (!SrcLocInfo) {
8792 uint32_t SrcLocStrSize;
8793 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8794 SrcLocInfo = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8795 }
8796
8797 Value *OffloadingArgs[] = {SrcLocInfo, DeviceID,
8798 PointerNum, RTArgs.BasePointersArray,
8799 RTArgs.PointersArray, RTArgs.SizesArray,
8800 RTArgs.MapTypesArray, RTArgs.MapNamesArray,
8801 RTArgs.MappersArray};
8802 Function *EndMapperFunc =
8803 getOrCreateRuntimeFunctionPtr(FnID: omp::OMPRTL___tgt_target_data_end_mapper);
8804
8805 createRuntimeFunctionCall(Callee: EndMapperFunc, Args: OffloadingArgs);
8806 return Error::success();
8807 };
8808
8809 // We don't have to do anything to close the region if the if clause evaluates
8810 // to false.
8811 auto EndElseGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
8812 ArrayRef<BasicBlock *> DeallocBlocks) {
8813 return Error::success();
8814 };
8815
8816 Error Err = [&]() -> Error {
8817 if (BodyGenCB) {
8818 Error Err = [&]() {
8819 if (IfCond)
8820 return emitIfClause(Cond: IfCond, ThenGen: BeginThenGen, ElseGen: BeginElseGen, AllocaIP);
8821 return BeginThenGen(AllocaIP, Builder.saveIP(), DeallocBlocks);
8822 }();
8823
8824 if (Err)
8825 return Err;
8826
8827 // If we don't require privatization of device pointers, we emit the body
8828 // in between the runtime calls. This avoids duplicating the body code.
8829 InsertPointOrErrorTy AfterIP =
8830 BodyGenCB(Builder.saveIP(), BodyGenTy::NoPriv);
8831 if (!AfterIP)
8832 return AfterIP.takeError();
8833 restoreIPandDebugLoc(Builder, IP: *AfterIP);
8834
8835 if (IfCond)
8836 return emitIfClause(Cond: IfCond, ThenGen: EndThenGen, ElseGen: EndElseGen, AllocaIP);
8837 return EndThenGen(AllocaIP, Builder.saveIP(), DeallocBlocks);
8838 }
8839 if (IfCond)
8840 return emitIfClause(Cond: IfCond, ThenGen: BeginThenGen, ElseGen: EndElseGen, AllocaIP);
8841 return BeginThenGen(AllocaIP, Builder.saveIP(), DeallocBlocks);
8842 }();
8843
8844 if (Err)
8845 return Err;
8846
8847 return Builder.saveIP();
8848}
8849
8850FunctionCallee
8851OpenMPIRBuilder::createForStaticInitFunction(unsigned IVSize, bool IVSigned,
8852 bool IsGPUDistribute) {
8853 assert((IVSize == 32 || IVSize == 64) &&
8854 "IV size is not compatible with the omp runtime");
8855 RuntimeFunction Name;
8856 if (IsGPUDistribute)
8857 Name = IVSize == 32
8858 ? (IVSigned ? omp::OMPRTL___kmpc_distribute_static_init_4
8859 : omp::OMPRTL___kmpc_distribute_static_init_4u)
8860 : (IVSigned ? omp::OMPRTL___kmpc_distribute_static_init_8
8861 : omp::OMPRTL___kmpc_distribute_static_init_8u);
8862 else
8863 Name = IVSize == 32 ? (IVSigned ? omp::OMPRTL___kmpc_for_static_init_4
8864 : omp::OMPRTL___kmpc_for_static_init_4u)
8865 : (IVSigned ? omp::OMPRTL___kmpc_for_static_init_8
8866 : omp::OMPRTL___kmpc_for_static_init_8u);
8867
8868 return getOrCreateRuntimeFunction(M, FnID: Name);
8869}
8870
8871FunctionCallee OpenMPIRBuilder::createDispatchInitFunction(unsigned IVSize,
8872 bool IVSigned) {
8873 assert((IVSize == 32 || IVSize == 64) &&
8874 "IV size is not compatible with the omp runtime");
8875 RuntimeFunction Name = IVSize == 32
8876 ? (IVSigned ? omp::OMPRTL___kmpc_dispatch_init_4
8877 : omp::OMPRTL___kmpc_dispatch_init_4u)
8878 : (IVSigned ? omp::OMPRTL___kmpc_dispatch_init_8
8879 : omp::OMPRTL___kmpc_dispatch_init_8u);
8880
8881 return getOrCreateRuntimeFunction(M, FnID: Name);
8882}
8883
8884FunctionCallee OpenMPIRBuilder::createDispatchNextFunction(unsigned IVSize,
8885 bool IVSigned) {
8886 assert((IVSize == 32 || IVSize == 64) &&
8887 "IV size is not compatible with the omp runtime");
8888 RuntimeFunction Name = IVSize == 32
8889 ? (IVSigned ? omp::OMPRTL___kmpc_dispatch_next_4
8890 : omp::OMPRTL___kmpc_dispatch_next_4u)
8891 : (IVSigned ? omp::OMPRTL___kmpc_dispatch_next_8
8892 : omp::OMPRTL___kmpc_dispatch_next_8u);
8893
8894 return getOrCreateRuntimeFunction(M, FnID: Name);
8895}
8896
8897FunctionCallee OpenMPIRBuilder::createDispatchFiniFunction(unsigned IVSize,
8898 bool IVSigned) {
8899 assert((IVSize == 32 || IVSize == 64) &&
8900 "IV size is not compatible with the omp runtime");
8901 RuntimeFunction Name = IVSize == 32
8902 ? (IVSigned ? omp::OMPRTL___kmpc_dispatch_fini_4
8903 : omp::OMPRTL___kmpc_dispatch_fini_4u)
8904 : (IVSigned ? omp::OMPRTL___kmpc_dispatch_fini_8
8905 : omp::OMPRTL___kmpc_dispatch_fini_8u);
8906
8907 return getOrCreateRuntimeFunction(M, FnID: Name);
8908}
8909
8910FunctionCallee OpenMPIRBuilder::createDispatchDeinitFunction() {
8911 return getOrCreateRuntimeFunction(M, FnID: omp::OMPRTL___kmpc_dispatch_deinit);
8912}
8913
8914static void FixupDebugInfoForOutlinedFunction(
8915 OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, Function *Func,
8916 DenseMap<Value *, std::tuple<Value *, unsigned>> &ValueReplacementMap) {
8917
8918 DISubprogram *NewSP = Func->getSubprogram();
8919 if (!NewSP)
8920 return;
8921
8922 SmallDenseMap<DILocalVariable *, DILocalVariable *> RemappedVariables;
8923
8924 auto GetUpdatedDIVariable = [&](DILocalVariable *OldVar, unsigned arg) {
8925 DILocalVariable *&NewVar = RemappedVariables[OldVar];
8926 // Only use cached variable if the arg number matches. This is important
8927 // so that DIVariable created for privatized variables are not discarded.
8928 if (NewVar && (arg == NewVar->getArg()))
8929 return NewVar;
8930
8931 NewVar = llvm::DILocalVariable::get(
8932 Context&: Builder.getContext(), Scope: OldVar->getScope(), Name: OldVar->getName(),
8933 File: OldVar->getFile(), Line: OldVar->getLine(), Type: OldVar->getType(), Arg: arg,
8934 Flags: OldVar->getFlags(), AlignInBits: OldVar->getAlignInBits(), Annotations: OldVar->getAnnotations());
8935 return NewVar;
8936 };
8937
8938 auto UpdateDebugRecord = [&](auto *DR) {
8939 DILocalVariable *OldVar = DR->getVariable();
8940 unsigned ArgNo = 0;
8941 for (auto Loc : DR->location_ops()) {
8942 auto Iter = ValueReplacementMap.find(Loc);
8943 if (Iter != ValueReplacementMap.end()) {
8944 DR->replaceVariableLocationOp(Loc, std::get<0>(Iter->second));
8945 ArgNo = std::get<1>(Iter->second) + 1;
8946 }
8947 }
8948 if (ArgNo != 0)
8949 DR->setVariable(GetUpdatedDIVariable(OldVar, ArgNo));
8950 };
8951
8952 SmallVector<DbgVariableRecord *, 4> DVRsToDelete;
8953 auto MoveDebugRecordToCorrectBlock = [&](DbgVariableRecord *DVR) {
8954 if (DVR->getNumVariableLocationOps() != 1u) {
8955 DVR->setKillLocation();
8956 return;
8957 }
8958 Value *Loc = DVR->getVariableLocationOp(OpIdx: 0u);
8959 BasicBlock *CurBB = DVR->getParent();
8960 BasicBlock *RequiredBB = nullptr;
8961
8962 if (Instruction *LocInst = dyn_cast<Instruction>(Val: Loc))
8963 RequiredBB = LocInst->getParent();
8964 else if (isa<llvm::Argument>(Val: Loc))
8965 RequiredBB = &DVR->getFunction()->getEntryBlock();
8966
8967 if (RequiredBB && RequiredBB != CurBB) {
8968 assert(!RequiredBB->empty());
8969 RequiredBB->insertDbgRecordBefore(DR: DVR->clone(),
8970 Here: RequiredBB->back().getIterator());
8971 DVRsToDelete.push_back(Elt: DVR);
8972 }
8973 };
8974
8975 // The location and scope of variable intrinsics and records still point to
8976 // the parent function of the target region. Update them.
8977 for (Instruction &I : instructions(F: Func)) {
8978 assert(!isa<llvm::DbgVariableIntrinsic>(&I) &&
8979 "Unexpected debug intrinsic");
8980 for (DbgVariableRecord &DVR : filterDbgVars(R: I.getDbgRecordRange())) {
8981 UpdateDebugRecord(&DVR);
8982 MoveDebugRecordToCorrectBlock(&DVR);
8983 }
8984 }
8985 for (auto *DVR : DVRsToDelete)
8986 DVR->getMarker()->MarkedInstr->dropOneDbgRecord(I: DVR);
8987 // An extra argument is passed to the device. Create the debug data for it.
8988 if (OMPBuilder.Config.isTargetDevice()) {
8989 DICompileUnit *CU = NewSP->getUnit();
8990 Module *M = Func->getParent();
8991 DIBuilder DB(*M, true, CU);
8992 DIType *VoidPtrTy =
8993 DB.createQualifiedType(Tag: dwarf::DW_TAG_pointer_type, FromTy: nullptr);
8994 unsigned ArgNo = Func->arg_size();
8995 DILocalVariable *Var = DB.createParameterVariable(
8996 Scope: NewSP, Name: "dyn_ptr", ArgNo, File: NewSP->getFile(), /*LineNo=*/0, Ty: VoidPtrTy,
8997 /*AlwaysPreserve=*/false, Flags: DINode::DIFlags::FlagArtificial);
8998 auto Loc = DILocation::get(Context&: Func->getContext(), Line: 0, Column: 0, Scope: NewSP, InlinedAt: 0);
8999 Argument *LastArg = Func->getArg(i: Func->arg_size() - 1);
9000 DB.insertDeclare(Storage: LastArg, VarInfo: Var, Expr: DB.createExpression(), DL: Loc,
9001 InsertAtEnd: &(*Func->begin()));
9002 }
9003}
9004
9005static Value *removeASCastIfPresent(Value *V) {
9006 if (Operator::getOpcode(V) == Instruction::AddrSpaceCast)
9007 return cast<Operator>(Val: V)->getOperand(i: 0);
9008 return V;
9009}
9010
9011static Expected<Function *> createOutlinedFunction(
9012 OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder,
9013 const OpenMPIRBuilder::TargetKernelDefaultAttrs &DefaultAttrs,
9014 StringRef FuncName, SmallVectorImpl<Value *> &Inputs,
9015 OpenMPIRBuilder::TargetBodyGenCallbackTy &CBFunc,
9016 OpenMPIRBuilder::TargetGenArgAccessorsCallbackTy &ArgAccessorFuncCB) {
9017 SmallVector<Type *> ParameterTypes;
9018 if (OMPBuilder.Config.isTargetDevice()) {
9019 // All parameters to target devices are passed as pointers
9020 // or i64. This assumes 64-bit address spaces/pointers.
9021 for (auto &Arg : Inputs)
9022 ParameterTypes.push_back(Elt: Arg->getType()->isPointerTy()
9023 ? Arg->getType()
9024 : Type::getInt64Ty(C&: Builder.getContext()));
9025 } else {
9026 for (auto &Arg : Inputs)
9027 ParameterTypes.push_back(Elt: Arg->getType());
9028 }
9029
9030 // The implicit dyn_ptr argument is always the last parameter on both host
9031 // and device so the argument counts match without runtime manipulation.
9032 auto *PtrTy = PointerType::getUnqual(C&: Builder.getContext());
9033 ParameterTypes.push_back(Elt: PtrTy);
9034
9035 auto BB = Builder.GetInsertBlock();
9036 auto M = BB->getModule();
9037 auto FuncType = FunctionType::get(Result: Builder.getVoidTy(), Params: ParameterTypes,
9038 /*isVarArg*/ false);
9039 auto Func =
9040 Function::Create(Ty: FuncType, Linkage: GlobalValue::InternalLinkage, N: FuncName, M);
9041
9042 // Forward target-cpu and target-features function attributes from the
9043 // original function to the new outlined function.
9044 Function *ParentFn = Builder.GetInsertBlock()->getParent();
9045
9046 auto TargetCpuAttr = ParentFn->getFnAttribute(Kind: "target-cpu");
9047 if (TargetCpuAttr.isStringAttribute())
9048 Func->addFnAttr(Attr: TargetCpuAttr);
9049
9050 auto TargetFeaturesAttr = ParentFn->getFnAttribute(Kind: "target-features");
9051 if (TargetFeaturesAttr.isStringAttribute())
9052 Func->addFnAttr(Attr: TargetFeaturesAttr);
9053
9054 if (OMPBuilder.Config.isTargetDevice()) {
9055 Value *ExecMode =
9056 OMPBuilder.emitKernelExecutionMode(KernelName: FuncName, Mode: DefaultAttrs.ExecFlags);
9057 OMPBuilder.emitUsed(Name: "llvm.compiler.used", List: {ExecMode});
9058 }
9059
9060 // Save insert point.
9061 IRBuilder<>::InsertPointGuard IPG(Builder);
9062 // We will generate the entries in the outlined function but the debug
9063 // location may still be pointing to the parent function. Reset it now.
9064 Builder.SetCurrentDebugLocation(llvm::DebugLoc());
9065
9066 // Generate the region into the function.
9067 BasicBlock *EntryBB = BasicBlock::Create(Context&: Builder.getContext(), Name: "entry", Parent: Func);
9068 Builder.SetInsertPoint(EntryBB);
9069
9070 // Insert target init call in the device compilation pass.
9071 if (OMPBuilder.Config.isTargetDevice())
9072 Builder.restoreIP(IP: OMPBuilder.createTargetInit(Loc: Builder, Attrs: DefaultAttrs));
9073
9074 BasicBlock *UserCodeEntryBB = Builder.GetInsertBlock();
9075
9076 // As we embed the user code in the middle of our target region after we
9077 // generate entry code, we must move what allocas we can into the entry
9078 // block to avoid possible breaking optimisations for device
9079 if (OMPBuilder.Config.isTargetDevice())
9080 OMPBuilder.ConstantAllocaRaiseCandidates.emplace_back(Args&: Func);
9081
9082 BasicBlock *ExitBB = splitBB(Builder, /*CreateBranch=*/true, Name: "target.exit");
9083 BasicBlock *OutlinedBodyBB =
9084 splitBB(Builder, /*CreateBranch=*/true, Name: "outlined.body");
9085 llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = CBFunc(
9086 Builder.saveIP(),
9087 OpenMPIRBuilder::InsertPointTy(OutlinedBodyBB, OutlinedBodyBB->begin()),
9088 ExitBB);
9089 if (!AfterIP)
9090 return AfterIP.takeError();
9091 Builder.SetInsertPoint(ExitBB);
9092
9093 // Insert target deinit call in the device compilation pass.
9094 if (OMPBuilder.Config.isTargetDevice())
9095 OMPBuilder.createTargetDeinit(Loc: Builder);
9096
9097 // Insert return instruction.
9098 Builder.CreateRetVoid();
9099
9100 // New Alloca IP at entry point of created device function.
9101 Builder.SetInsertPoint(EntryBB->getFirstNonPHIIt());
9102 auto AllocaIP = Builder.saveIP();
9103
9104 Builder.SetInsertPoint(UserCodeEntryBB->getFirstNonPHIOrDbg());
9105
9106 // Do not include the artificial dyn_ptr argument.
9107 const auto &ArgRange = make_range(x: Func->arg_begin(), y: Func->arg_end() - 1);
9108
9109 DenseMap<Value *, std::tuple<Value *, unsigned>> ValueReplacementMap;
9110
9111 auto ReplaceValue = [](Value *Input, Value *InputCopy, Function *Func) {
9112 // Things like GEP's can come in the form of Constants. Constants and
9113 // ConstantExpr's do not have access to the knowledge of what they're
9114 // contained in, so we must dig a little to find an instruction so we
9115 // can tell if they're used inside of the function we're outlining. We
9116 // also replace the original constant expression with a new instruction
9117 // equivalent; an instruction as it allows easy modification in the
9118 // following loop, as we can now know the constant (instruction) is
9119 // owned by our target function and replaceUsesOfWith can now be invoked
9120 // on it (cannot do this with constants it seems). A brand new one also
9121 // allows us to be cautious as it is perhaps possible the old expression
9122 // was used inside of the function but exists and is used externally
9123 // (unlikely by the nature of a Constant, but still).
9124 // NOTE: We cannot remove dead constants that have been rewritten to
9125 // instructions at this stage, we run the risk of breaking later lowering
9126 // by doing so as we could still be in the process of lowering the module
9127 // from MLIR to LLVM-IR and the MLIR lowering may still require the original
9128 // constants we have created rewritten versions of.
9129 if (auto *Const = dyn_cast<Constant>(Val: Input))
9130 convertUsersOfConstantsToInstructions(Consts: Const, RestrictToFunc: Func, RemoveDeadConstants: false);
9131
9132 // Collect users before iterating over them to avoid invalidating the
9133 // iteration in case a user uses Input more than once (e.g. a call
9134 // instruction).
9135 SetVector<User *> Users(Input->users().begin(), Input->users().end());
9136 // Collect all the instructions
9137 for (User *User : make_early_inc_range(Range&: Users))
9138 if (auto *Instr = dyn_cast<Instruction>(Val: User))
9139 if (Instr->getFunction() == Func)
9140 Instr->replaceUsesOfWith(From: Input, To: InputCopy);
9141 };
9142
9143 SmallVector<std::pair<Value *, Value *>> DeferredReplacement;
9144
9145 // Rewrite uses of input valus to parameters.
9146 for (auto InArg : zip(t&: Inputs, u: ArgRange)) {
9147 Value *Input = std::get<0>(t&: InArg);
9148 Argument &Arg = std::get<1>(t&: InArg);
9149 Value *InputCopy = nullptr;
9150
9151 llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = ArgAccessorFuncCB(
9152 Arg, Input, InputCopy, AllocaIP, Builder.saveIP(),
9153 OpenMPIRBuilder::InsertPointTy(ExitBB, ExitBB->begin()));
9154 if (!AfterIP)
9155 return AfterIP.takeError();
9156 Builder.restoreIP(IP: *AfterIP);
9157 ValueReplacementMap[Input] = std::make_tuple(args&: InputCopy, args: Arg.getArgNo());
9158
9159 // In certain cases a Global may be set up for replacement, however, this
9160 // Global may be used in multiple arguments to the kernel, just segmented
9161 // apart, for example, if we have a global array, that is sectioned into
9162 // multiple mappings (technically not legal in OpenMP, but there is a case
9163 // in Fortran for Common Blocks where this is neccesary), we will end up
9164 // with GEP's into this array inside the kernel, that refer to the Global
9165 // but are technically separate arguments to the kernel for all intents and
9166 // purposes. If we have mapped a segment that requires a GEP into the 0-th
9167 // index, it will fold into an referal to the Global, if we then encounter
9168 // this folded GEP during replacement all of the references to the
9169 // Global in the kernel will be replaced with the argument we have generated
9170 // that corresponds to it, including any other GEP's that refer to the
9171 // Global that may be other arguments. This will invalidate all of the other
9172 // preceding mapped arguments that refer to the same global that may be
9173 // separate segments. To prevent this, we defer global processing until all
9174 // other processing has been performed.
9175 if (llvm::isa<llvm::GlobalValue, llvm::GlobalObject, llvm::GlobalVariable>(
9176 Val: removeASCastIfPresent(V: Input))) {
9177 DeferredReplacement.push_back(Elt: std::make_pair(x&: Input, y&: InputCopy));
9178 continue;
9179 }
9180
9181 if (isa<ConstantData>(Val: Input))
9182 continue;
9183
9184 ReplaceValue(Input, InputCopy, Func);
9185 }
9186
9187 // Replace all of our deferred Input values, currently just Globals.
9188 for (auto Deferred : DeferredReplacement)
9189 ReplaceValue(std::get<0>(in&: Deferred), std::get<1>(in&: Deferred), Func);
9190
9191 FixupDebugInfoForOutlinedFunction(OMPBuilder, Builder, Func,
9192 ValueReplacementMap);
9193 return Func;
9194}
9195/// Given a task descriptor, TaskWithPrivates, return the pointer to the block
9196/// of pointers containing shared data between the parent task and the created
9197/// task.
9198static LoadInst *loadSharedDataFromTaskDescriptor(OpenMPIRBuilder &OMPIRBuilder,
9199 IRBuilderBase &Builder,
9200 Value *TaskWithPrivates,
9201 Type *TaskWithPrivatesTy) {
9202
9203 Type *TaskTy = OMPIRBuilder.Task;
9204 LLVMContext &Ctx = Builder.getContext();
9205 Value *TaskT =
9206 Builder.CreateStructGEP(Ty: TaskWithPrivatesTy, Ptr: TaskWithPrivates, Idx: 0);
9207 Value *Shareds = TaskT;
9208 // TaskWithPrivatesTy can be one of the following
9209 // 1. %struct.task_with_privates = type { %struct.kmp_task_ompbuilder_t,
9210 // %struct.privates }
9211 // 2. %struct.kmp_task_ompbuilder_t ;; This is simply TaskTy
9212 //
9213 // In the former case, that is when TaskWithPrivatesTy != TaskTy,
9214 // its first member has to be the task descriptor. TaskTy is the type of the
9215 // task descriptor. TaskT is the pointer to the task descriptor. Loading the
9216 // first member of TaskT, gives us the pointer to shared data.
9217 if (TaskWithPrivatesTy != TaskTy)
9218 Shareds = Builder.CreateStructGEP(Ty: TaskTy, Ptr: TaskT, Idx: 0);
9219 return Builder.CreateLoad(Ty: PointerType::getUnqual(C&: Ctx), Ptr: Shareds);
9220}
9221/// Create an entry point for a target task with the following.
9222/// It'll have the following signature
9223/// void @.omp_target_task_proxy_func(i32 %thread.id, ptr %task)
9224/// This function is called from emitTargetTask once the
9225/// code to launch the target kernel has been outlined already.
9226/// NumOffloadingArrays is the number of offloading arrays that we need to copy
9227/// into the task structure so that the deferred target task can access this
9228/// data even after the stack frame of the generating task has been rolled
9229/// back. Offloading arrays contain base pointers, pointers, sizes etc
9230/// of the data that the target kernel will access. These in effect are the
9231/// non-empty arrays of pointers held by OpenMPIRBuilder::TargetDataRTArgs.
9232static Function *emitTargetTaskProxyFunction(
9233 OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, CallInst *StaleCI,
9234 StructType *PrivatesTy, StructType *TaskWithPrivatesTy,
9235 const size_t NumOffloadingArrays, const int SharedArgsOperandNo) {
9236
9237 // If NumOffloadingArrays is non-zero, PrivatesTy better not be nullptr.
9238 // This is because PrivatesTy is the type of the structure in which
9239 // we pass the offloading arrays to the deferred target task.
9240 assert((!NumOffloadingArrays || PrivatesTy) &&
9241 "PrivatesTy cannot be nullptr when there are offloadingArrays"
9242 "to privatize");
9243
9244 Module &M = OMPBuilder.M;
9245 // KernelLaunchFunction is the target launch function, i.e.
9246 // the function that sets up kernel arguments and calls
9247 // __tgt_target_kernel to launch the kernel on the device.
9248 //
9249 Function *KernelLaunchFunction = StaleCI->getCalledFunction();
9250
9251 // StaleCI is the CallInst which is the call to the outlined
9252 // target kernel launch function. If there are local live-in values
9253 // that the outlined function uses then these are aggregated into a structure
9254 // which is passed as the second argument. If there are no local live-in
9255 // values or if all values used by the outlined kernel are global variables,
9256 // then there's only one argument, the threadID. So, StaleCI can be
9257 //
9258 // %structArg = alloca { ptr, ptr }, align 8
9259 // %gep_ = getelementptr { ptr, ptr }, ptr %structArg, i32 0, i32 0
9260 // store ptr %20, ptr %gep_, align 8
9261 // %gep_8 = getelementptr { ptr, ptr }, ptr %structArg, i32 0, i32 1
9262 // store ptr %21, ptr %gep_8, align 8
9263 // call void @_QQmain..omp_par.1(i32 %global.tid.val6, ptr %structArg)
9264 //
9265 // OR
9266 //
9267 // call void @_QQmain..omp_par.1(i32 %global.tid.val6)
9268 OpenMPIRBuilder::InsertPointTy IP(StaleCI->getParent(),
9269 StaleCI->getIterator());
9270
9271 LLVMContext &Ctx = StaleCI->getParent()->getContext();
9272
9273 Type *ThreadIDTy = Type::getInt32Ty(C&: Ctx);
9274 Type *TaskPtrTy = OMPBuilder.TaskPtr;
9275 [[maybe_unused]] Type *TaskTy = OMPBuilder.Task;
9276
9277 auto ProxyFnTy =
9278 FunctionType::get(Result: Builder.getVoidTy(), Params: {ThreadIDTy, TaskPtrTy},
9279 /* isVarArg */ false);
9280 auto ProxyFn = Function::Create(Ty: ProxyFnTy, Linkage: GlobalValue::InternalLinkage,
9281 N: ".omp_target_task_proxy_func",
9282 M: Builder.GetInsertBlock()->getModule());
9283 Value *ThreadId = ProxyFn->getArg(i: 0);
9284 Value *TaskWithPrivates = ProxyFn->getArg(i: 1);
9285 ThreadId->setName("thread.id");
9286 TaskWithPrivates->setName("task");
9287
9288 bool HasShareds = SharedArgsOperandNo > 0;
9289 bool HasOffloadingArrays = NumOffloadingArrays > 0;
9290 BasicBlock *EntryBB =
9291 BasicBlock::Create(Context&: Builder.getContext(), Name: "entry", Parent: ProxyFn);
9292 Builder.SetInsertPoint(EntryBB);
9293
9294 SmallVector<Value *> KernelLaunchArgs;
9295 KernelLaunchArgs.reserve(N: StaleCI->arg_size());
9296 KernelLaunchArgs.push_back(Elt: ThreadId);
9297
9298 if (HasOffloadingArrays) {
9299 assert(TaskTy != TaskWithPrivatesTy &&
9300 "If there are offloading arrays to pass to the target"
9301 "TaskTy cannot be the same as TaskWithPrivatesTy");
9302 (void)TaskTy;
9303 Value *Privates =
9304 Builder.CreateStructGEP(Ty: TaskWithPrivatesTy, Ptr: TaskWithPrivates, Idx: 1);
9305 for (unsigned int i = 0; i < NumOffloadingArrays; ++i)
9306 KernelLaunchArgs.push_back(
9307 Elt: Builder.CreateStructGEP(Ty: PrivatesTy, Ptr: Privates, Idx: i));
9308 }
9309
9310 if (HasShareds) {
9311 auto *ArgStructAlloca =
9312 dyn_cast<AllocaInst>(Val: StaleCI->getArgOperand(i: SharedArgsOperandNo));
9313 assert(ArgStructAlloca &&
9314 "Unable to find the alloca instruction corresponding to arguments "
9315 "for extracted function");
9316 auto *ArgStructType = cast<StructType>(Val: ArgStructAlloca->getAllocatedType());
9317 std::optional<TypeSize> ArgAllocSize =
9318 ArgStructAlloca->getAllocationSize(DL: M.getDataLayout());
9319 assert(ArgStructType && ArgAllocSize &&
9320 "Unable to determine size of arguments for extracted function");
9321 uint64_t StructSize = ArgAllocSize->getFixedValue();
9322
9323 AllocaInst *NewArgStructAlloca =
9324 Builder.CreateAlloca(Ty: ArgStructType, ArraySize: nullptr, Name: "structArg");
9325
9326 Value *SharedsSize = Builder.getInt64(C: StructSize);
9327
9328 LoadInst *LoadShared = loadSharedDataFromTaskDescriptor(
9329 OMPIRBuilder&: OMPBuilder, Builder, TaskWithPrivates, TaskWithPrivatesTy);
9330
9331 Builder.CreateMemCpy(
9332 Dst: NewArgStructAlloca, DstAlign: NewArgStructAlloca->getAlign(), Src: LoadShared,
9333 SrcAlign: LoadShared->getPointerAlignment(DL: M.getDataLayout()), Size: SharedsSize);
9334 KernelLaunchArgs.push_back(Elt: NewArgStructAlloca);
9335 }
9336 OMPBuilder.createRuntimeFunctionCall(Callee: KernelLaunchFunction, Args: KernelLaunchArgs);
9337 Builder.CreateRetVoid();
9338 return ProxyFn;
9339}
9340static Type *getOffloadingArrayType(Value *V) {
9341
9342 if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: V))
9343 return GEP->getSourceElementType();
9344 if (auto *Alloca = dyn_cast<AllocaInst>(Val: V))
9345 return Alloca->getAllocatedType();
9346
9347 llvm_unreachable("Unhandled Instruction type");
9348 return nullptr;
9349}
9350// This function returns a struct that has at most two members.
9351// The first member is always %struct.kmp_task_ompbuilder_t, that is the task
9352// descriptor. The second member, if needed, is a struct containing arrays
9353// that need to be passed to the offloaded target kernel. For example,
9354// if .offload_baseptrs, .offload_ptrs and .offload_sizes have to be passed to
9355// the target kernel and their types are [3 x ptr], [3 x ptr] and [3 x i64]
9356// respectively, then the types created by this function are
9357//
9358// %struct.privates = type { [3 x ptr], [3 x ptr], [3 x i64] }
9359// %struct.task_with_privates = type { %struct.kmp_task_ompbuilder_t,
9360// %struct.privates }
9361// %struct.task_with_privates is returned by this function.
9362// If there aren't any offloading arrays to pass to the target kernel,
9363// %struct.kmp_task_ompbuilder_t is returned.
9364static StructType *
9365createTaskWithPrivatesTy(OpenMPIRBuilder &OMPIRBuilder,
9366 ArrayRef<Value *> OffloadingArraysToPrivatize) {
9367
9368 if (OffloadingArraysToPrivatize.empty())
9369 return OMPIRBuilder.Task;
9370
9371 SmallVector<Type *, 4> StructFieldTypes;
9372 for (Value *V : OffloadingArraysToPrivatize) {
9373 assert(V->getType()->isPointerTy() &&
9374 "Expected pointer to array to privatize. Got a non-pointer value "
9375 "instead");
9376 Type *ArrayTy = getOffloadingArrayType(V);
9377 assert(ArrayTy && "ArrayType cannot be nullptr");
9378 StructFieldTypes.push_back(Elt: ArrayTy);
9379 }
9380 StructType *PrivatesStructTy =
9381 StructType::create(Elements: StructFieldTypes, Name: "struct.privates");
9382 return StructType::create(Elements: {OMPIRBuilder.Task, PrivatesStructTy},
9383 Name: "struct.task_with_privates");
9384}
9385static Error emitTargetOutlinedFunction(
9386 OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, bool IsOffloadEntry,
9387 TargetRegionEntryInfo &EntryInfo,
9388 const OpenMPIRBuilder::TargetKernelDefaultAttrs &DefaultAttrs,
9389 Function *&OutlinedFn, Constant *&OutlinedFnID,
9390 SmallVectorImpl<Value *> &Inputs,
9391 OpenMPIRBuilder::TargetBodyGenCallbackTy &CBFunc,
9392 OpenMPIRBuilder::TargetGenArgAccessorsCallbackTy &ArgAccessorFuncCB) {
9393
9394 OpenMPIRBuilder::FunctionGenCallback &&GenerateOutlinedFunction =
9395 [&](StringRef EntryFnName) {
9396 return createOutlinedFunction(OMPBuilder, Builder, DefaultAttrs,
9397 FuncName: EntryFnName, Inputs, CBFunc,
9398 ArgAccessorFuncCB);
9399 };
9400
9401 return OMPBuilder.emitTargetRegionFunction(
9402 EntryInfo, GenerateFunctionCallback&: GenerateOutlinedFunction, IsOffloadEntry, OutlinedFn,
9403 OutlinedFnID);
9404}
9405
9406OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitTargetTask(
9407 TargetTaskBodyCallbackTy TaskBodyCB, Value *DeviceID, Value *RTLoc,
9408 OpenMPIRBuilder::InsertPointTy AllocaIP,
9409 const DependenciesInfo &Dependencies, const TargetDataRTArgs &RTArgs,
9410 bool HasNoWait) {
9411
9412 // The following explains the code-gen scenario for the `target` directive. A
9413 // similar scneario is followed for other device-related directives (e.g.
9414 // `target enter data`) but in similar fashion since we only need to emit task
9415 // that encapsulates the proper runtime call.
9416 //
9417 // When we arrive at this function, the target region itself has been
9418 // outlined into the function OutlinedFn.
9419 // So at ths point, for
9420 // --------------------------------------------------------------
9421 // void user_code_that_offloads(...) {
9422 // omp target depend(..) map(from:a) map(to:b) private(i)
9423 // do i = 1, 10
9424 // a(i) = b(i) + n
9425 // }
9426 //
9427 // --------------------------------------------------------------
9428 //
9429 // we have
9430 //
9431 // --------------------------------------------------------------
9432 //
9433 // void user_code_that_offloads(...) {
9434 // %.offload_baseptrs = alloca [2 x ptr], align 8
9435 // %.offload_ptrs = alloca [2 x ptr], align 8
9436 // %.offload_mappers = alloca [2 x ptr], align 8
9437 // ;; target region has been outlined and now we need to
9438 // ;; offload to it via a target task.
9439 // }
9440 // void outlined_device_function(ptr a, ptr b, ptr n) {
9441 // n = *n_ptr;
9442 // do i = 1, 10
9443 // a(i) = b(i) + n
9444 // }
9445 //
9446 // We have to now do the following
9447 // (i) Make an offloading call to outlined_device_function using the OpenMP
9448 // RTL. See 'kernel_launch_function' in the pseudo code below. This is
9449 // emitted by emitKernelLaunch
9450 // (ii) Create a task entry point function that calls kernel_launch_function
9451 // and is the entry point for the target task. See
9452 // '@.omp_target_task_proxy_func in the pseudocode below.
9453 // (iii) Create a task with the task entry point created in (ii)
9454 //
9455 // That is we create the following
9456 // struct task_with_privates {
9457 // struct kmp_task_ompbuilder_t task_struct;
9458 // struct privates {
9459 // [2 x ptr] ; baseptrs
9460 // [2 x ptr] ; ptrs
9461 // [2 x i64] ; sizes
9462 // }
9463 // }
9464 // void user_code_that_offloads(...) {
9465 // %.offload_baseptrs = alloca [2 x ptr], align 8
9466 // %.offload_ptrs = alloca [2 x ptr], align 8
9467 // %.offload_sizes = alloca [2 x i64], align 8
9468 //
9469 // %structArg = alloca { ptr, ptr, ptr }, align 8
9470 // %strucArg[0] = a
9471 // %strucArg[1] = b
9472 // %strucArg[2] = &n
9473 //
9474 // target_task_with_privates = @__kmpc_omp_target_task_alloc(...,
9475 // sizeof(kmp_task_ompbuilder_t),
9476 // sizeof(structArg),
9477 // @.omp_target_task_proxy_func,
9478 // ...)
9479 // memcpy(target_task_with_privates->task_struct->shareds, %structArg,
9480 // sizeof(structArg))
9481 // memcpy(target_task_with_privates->privates->baseptrs,
9482 // offload_baseptrs, sizeof(offload_baseptrs)
9483 // memcpy(target_task_with_privates->privates->ptrs,
9484 // offload_ptrs, sizeof(offload_ptrs)
9485 // memcpy(target_task_with_privates->privates->sizes,
9486 // offload_sizes, sizeof(offload_sizes)
9487 // dependencies_array = ...
9488 // ;; if nowait not present
9489 // call @__kmpc_omp_wait_deps(..., dependencies_array)
9490 // call @__kmpc_omp_task_begin_if0(...)
9491 // call @ @.omp_target_task_proxy_func(i32 thread_id, ptr
9492 // %target_task_with_privates)
9493 // call @__kmpc_omp_task_complete_if0(...)
9494 // }
9495 //
9496 // define internal void @.omp_target_task_proxy_func(i32 %thread.id,
9497 // ptr %task) {
9498 // %structArg = alloca {ptr, ptr, ptr}
9499 // %task_ptr = getelementptr(%task, 0, 0)
9500 // %shared_data = load (getelementptr %task_ptr, 0, 0)
9501 // mempcy(%structArg, %shared_data, sizeof(%structArg))
9502 //
9503 // %offloading_arrays = getelementptr(%task, 0, 1)
9504 // %offload_baseptrs = getelementptr(%offloading_arrays, 0, 0)
9505 // %offload_ptrs = getelementptr(%offloading_arrays, 0, 1)
9506 // %offload_sizes = getelementptr(%offloading_arrays, 0, 2)
9507 // kernel_launch_function(%thread.id, %offload_baseptrs, %offload_ptrs,
9508 // %offload_sizes, %structArg)
9509 // }
9510 //
9511 // We need the proxy function because the signature of the task entry point
9512 // expected by kmpc_omp_task is always the same and will be different from
9513 // that of the kernel_launch function.
9514 //
9515 // kernel_launch_function is generated by emitKernelLaunch and has the
9516 // always_inline attribute. For this example, it'll look like so:
9517 // void kernel_launch_function(%thread_id, %offload_baseptrs, %offload_ptrs,
9518 // %offload_sizes, %structArg) alwaysinline {
9519 // %kernel_args = alloca %struct.__tgt_kernel_arguments, align 8
9520 // ; load aggregated data from %structArg
9521 // ; setup kernel_args using offload_baseptrs, offload_ptrs and
9522 // ; offload_sizes
9523 // call i32 @__tgt_target_kernel(...,
9524 // outlined_device_function,
9525 // ptr %kernel_args)
9526 // }
9527 // void outlined_device_function(ptr a, ptr b, ptr n) {
9528 // n = *n_ptr;
9529 // do i = 1, 10
9530 // a(i) = b(i) + n
9531 // }
9532 //
9533 BasicBlock *TargetTaskBodyBB =
9534 splitBB(Builder, /*CreateBranch=*/true, Name: "target.task.body");
9535 BasicBlock *TargetTaskAllocaBB =
9536 splitBB(Builder, /*CreateBranch=*/true, Name: "target.task.alloca");
9537
9538 InsertPointTy TargetTaskAllocaIP(TargetTaskAllocaBB,
9539 TargetTaskAllocaBB->begin());
9540 InsertPointTy TargetTaskBodyIP(TargetTaskBodyBB, TargetTaskBodyBB->begin());
9541
9542 auto OI = std::make_unique<OutlineInfo>();
9543 OI->EntryBB = TargetTaskAllocaBB;
9544 OI->OuterAllocBB = AllocaIP.getBlock();
9545
9546 // Add the thread ID argument.
9547 SmallVector<Instruction *, 4> ToBeDeleted;
9548 OI->ExcludeArgsFromAggregate.push_back(Elt: createFakeIntVal(
9549 Builder, OuterAllocaIP: AllocaIP, ToBeDeleted, InnerAllocaIP: TargetTaskAllocaIP, Name: "global.tid", AsPtr: false));
9550
9551 // Generate the task body which will subsequently be outlined.
9552 Builder.restoreIP(IP: TargetTaskBodyIP);
9553 if (Error Err = TaskBodyCB(DeviceID, RTLoc, TargetTaskAllocaIP))
9554 return Err;
9555
9556 // The outliner (CodeExtractor) extract a sequence or vector of blocks that
9557 // it is given. These blocks are enumerated by
9558 // OpenMPIRBuilder::OutlineInfo::collectBlocks which expects the OI.ExitBlock
9559 // to be outside the region. In other words, OI.ExitBlock is expected to be
9560 // the start of the region after the outlining. We used to set OI.ExitBlock
9561 // to the InsertBlock after TaskBodyCB is done. This is fine in most cases
9562 // except when the task body is a single basic block. In that case,
9563 // OI.ExitBlock is set to the single task body block and will get left out of
9564 // the outlining process. So, simply create a new empty block to which we
9565 // uncoditionally branch from where TaskBodyCB left off
9566 OI->ExitBB = BasicBlock::Create(Context&: Builder.getContext(), Name: "target.task.cont");
9567 emitBlock(BB: OI->ExitBB, CurFn: Builder.GetInsertBlock()->getParent(),
9568 /*IsFinished=*/true);
9569
9570 SmallVector<Value *, 2> OffloadingArraysToPrivatize;
9571 bool NeedsTargetTask = HasNoWait && DeviceID;
9572 if (NeedsTargetTask) {
9573 for (auto *V :
9574 {RTArgs.BasePointersArray, RTArgs.PointersArray, RTArgs.MappersArray,
9575 RTArgs.MapNamesArray, RTArgs.MapTypesArray, RTArgs.MapTypesArrayEnd,
9576 RTArgs.SizesArray}) {
9577 if (V && !isa<ConstantPointerNull, GlobalVariable>(Val: V)) {
9578 OffloadingArraysToPrivatize.push_back(Elt: V);
9579 OI->ExcludeArgsFromAggregate.push_back(Elt: V);
9580 }
9581 }
9582 }
9583 OI->PostOutlineCB = [this, ToBeDeleted, Dependencies, NeedsTargetTask,
9584 DeviceID, OffloadingArraysToPrivatize](
9585 Function &OutlinedFn) mutable {
9586 assert(OutlinedFn.hasOneUse() &&
9587 "there must be a single user for the outlined function");
9588
9589 CallInst *StaleCI = cast<CallInst>(Val: OutlinedFn.user_back());
9590
9591 // The first argument of StaleCI is always the thread id.
9592 // The next few arguments are the pointers to offloading arrays
9593 // if any. (see OffloadingArraysToPrivatize)
9594 // Finally, all other local values that are live-in into the outlined region
9595 // end up in a structure whose pointer is passed as the last argument. This
9596 // piece of data is passed in the "shared" field of the task structure. So,
9597 // we know we have to pass shareds to the task if the number of arguments is
9598 // greater than OffloadingArraysToPrivatize.size() + 1 The 1 is for the
9599 // thread id. Further, for safety, we assert that the number of arguments of
9600 // StaleCI is exactly OffloadingArraysToPrivatize.size() + 2
9601 const unsigned int NumStaleCIArgs = StaleCI->arg_size();
9602 bool HasShareds = NumStaleCIArgs > OffloadingArraysToPrivatize.size() + 1;
9603 assert((!HasShareds ||
9604 NumStaleCIArgs == (OffloadingArraysToPrivatize.size() + 2)) &&
9605 "Wrong number of arguments for StaleCI when shareds are present");
9606 int SharedArgOperandNo =
9607 HasShareds ? OffloadingArraysToPrivatize.size() + 1 : 0;
9608
9609 StructType *TaskWithPrivatesTy =
9610 createTaskWithPrivatesTy(OMPIRBuilder&: *this, OffloadingArraysToPrivatize);
9611 StructType *PrivatesTy = nullptr;
9612
9613 if (!OffloadingArraysToPrivatize.empty())
9614 PrivatesTy =
9615 static_cast<StructType *>(TaskWithPrivatesTy->getElementType(N: 1));
9616
9617 Function *ProxyFn = emitTargetTaskProxyFunction(
9618 OMPBuilder&: *this, Builder, StaleCI, PrivatesTy, TaskWithPrivatesTy,
9619 NumOffloadingArrays: OffloadingArraysToPrivatize.size(), SharedArgsOperandNo: SharedArgOperandNo);
9620
9621 LLVM_DEBUG(dbgs() << "Proxy task entry function created: " << *ProxyFn
9622 << "\n");
9623
9624 Builder.SetInsertPoint(StaleCI);
9625
9626 // Gather the arguments for emitting the runtime call.
9627 uint32_t SrcLocStrSize;
9628 Constant *SrcLocStr =
9629 getOrCreateSrcLocStr(Loc: LocationDescription(Builder), SrcLocStrSize);
9630 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
9631
9632 // @__kmpc_omp_task_alloc or @__kmpc_omp_target_task_alloc
9633 //
9634 // If `HasNoWait == true`, we call @__kmpc_omp_target_task_alloc to provide
9635 // the DeviceID to the deferred task and also since
9636 // @__kmpc_omp_target_task_alloc creates an untied/async task.
9637 Function *TaskAllocFn =
9638 !NeedsTargetTask
9639 ? getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_alloc)
9640 : getOrCreateRuntimeFunctionPtr(
9641 FnID: OMPRTL___kmpc_omp_target_task_alloc);
9642
9643 // Arguments - `loc_ref` (Ident) and `gtid` (ThreadID)
9644 // call.
9645 Value *ThreadID = getOrCreateThreadID(Ident);
9646
9647 // Argument - `sizeof_kmp_task_t` (TaskSize)
9648 // Tasksize refers to the size in bytes of kmp_task_t data structure
9649 // plus any other data to be passed to the target task, if any, which
9650 // is packed into a struct. kmp_task_t and the struct so created are
9651 // packed into a wrapper struct whose type is TaskWithPrivatesTy.
9652 Value *TaskSize = Builder.getInt64(
9653 C: M.getDataLayout().getTypeStoreSize(Ty: TaskWithPrivatesTy));
9654
9655 // Argument - `sizeof_shareds` (SharedsSize)
9656 // SharedsSize refers to the shareds array size in the kmp_task_t data
9657 // structure.
9658 Value *SharedsSize = Builder.getInt64(C: 0);
9659 if (HasShareds) {
9660 auto *ArgStructAlloca =
9661 dyn_cast<AllocaInst>(Val: StaleCI->getArgOperand(i: SharedArgOperandNo));
9662 assert(ArgStructAlloca &&
9663 "Unable to find the alloca instruction corresponding to arguments "
9664 "for extracted function");
9665 std::optional<TypeSize> ArgAllocSize =
9666 ArgStructAlloca->getAllocationSize(DL: M.getDataLayout());
9667 assert(ArgAllocSize &&
9668 "Unable to determine size of arguments for extracted function");
9669 SharedsSize = Builder.getInt64(C: ArgAllocSize->getFixedValue());
9670 }
9671
9672 // Argument - `flags`
9673 // Task is tied iff (Flags & 1) == 1.
9674 // Task is untied iff (Flags & 1) == 0.
9675 // Task is final iff (Flags & 2) == 2.
9676 // Task is not final iff (Flags & 2) == 0.
9677 // A target task is not final and is untied.
9678 Value *Flags = Builder.getInt32(C: 0);
9679
9680 // Emit the @__kmpc_omp_task_alloc runtime call
9681 // The runtime call returns a pointer to an area where the task captured
9682 // variables must be copied before the task is run (TaskData)
9683 CallInst *TaskData = nullptr;
9684
9685 SmallVector<llvm::Value *> TaskAllocArgs = {
9686 /*loc_ref=*/Ident, /*gtid=*/ThreadID,
9687 /*flags=*/Flags,
9688 /*sizeof_task=*/TaskSize, /*sizeof_shared=*/SharedsSize,
9689 /*task_func=*/ProxyFn};
9690
9691 if (NeedsTargetTask) {
9692 assert(DeviceID && "Expected non-empty device ID.");
9693 TaskAllocArgs.push_back(Elt: DeviceID);
9694 }
9695
9696 TaskData = createRuntimeFunctionCall(Callee: TaskAllocFn, Args: TaskAllocArgs);
9697
9698 Align Alignment = TaskData->getPointerAlignment(DL: M.getDataLayout());
9699 if (HasShareds) {
9700 Value *Shareds = StaleCI->getArgOperand(i: SharedArgOperandNo);
9701 Value *TaskShareds = loadSharedDataFromTaskDescriptor(
9702 OMPIRBuilder&: *this, Builder, TaskWithPrivates: TaskData, TaskWithPrivatesTy);
9703 Builder.CreateMemCpy(Dst: TaskShareds, DstAlign: Alignment, Src: Shareds, SrcAlign: Alignment,
9704 Size: SharedsSize);
9705 }
9706 if (!OffloadingArraysToPrivatize.empty()) {
9707 Value *Privates =
9708 Builder.CreateStructGEP(Ty: TaskWithPrivatesTy, Ptr: TaskData, Idx: 1);
9709 for (unsigned int i = 0; i < OffloadingArraysToPrivatize.size(); ++i) {
9710 Value *PtrToPrivatize = OffloadingArraysToPrivatize[i];
9711 [[maybe_unused]] Type *ArrayType =
9712 getOffloadingArrayType(V: PtrToPrivatize);
9713 assert(ArrayType && "ArrayType cannot be nullptr");
9714
9715 Type *ElementType = PrivatesTy->getElementType(N: i);
9716 assert(ElementType == ArrayType &&
9717 "ElementType should match ArrayType");
9718 (void)ArrayType;
9719
9720 Value *Dst = Builder.CreateStructGEP(Ty: PrivatesTy, Ptr: Privates, Idx: i);
9721 Builder.CreateMemCpy(
9722 Dst, DstAlign: Alignment, Src: PtrToPrivatize, SrcAlign: Alignment,
9723 Size: Builder.getInt64(C: M.getDataLayout().getTypeStoreSize(Ty: ElementType)));
9724 }
9725 }
9726
9727 Value *DepArray = nullptr;
9728 Value *NumDeps = nullptr;
9729 if (Dependencies.DepArray) {
9730 DepArray = Dependencies.DepArray;
9731 NumDeps = Dependencies.NumDeps;
9732 } else if (!Dependencies.Deps.empty()) {
9733 DepArray = emitTaskDependencies(OMPBuilder&: *this, Dependencies: Dependencies.Deps);
9734 NumDeps = Builder.getInt32(C: Dependencies.Deps.size());
9735 }
9736
9737 // ---------------------------------------------------------------
9738 // V5.2 13.8 target construct
9739 // If the nowait clause is present, execution of the target task
9740 // may be deferred. If the nowait clause is not present, the target task is
9741 // an included task.
9742 // ---------------------------------------------------------------
9743 // The above means that the lack of a nowait on the target construct
9744 // translates to '#pragma omp task if(0)'
9745 if (!NeedsTargetTask) {
9746 if (DepArray) {
9747 Function *TaskWaitFn =
9748 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_wait_deps);
9749 createRuntimeFunctionCall(
9750 Callee: TaskWaitFn,
9751 Args: {/*loc_ref=*/Ident, /*gtid=*/ThreadID,
9752 /*ndeps=*/NumDeps,
9753 /*dep_list=*/DepArray,
9754 /*ndeps_noalias=*/ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0),
9755 /*noalias_dep_list=*/
9756 ConstantPointerNull::get(T: PointerType::getUnqual(C&: M.getContext()))});
9757 }
9758 // Included task.
9759 Function *TaskBeginFn =
9760 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_begin_if0);
9761 Function *TaskCompleteFn =
9762 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_complete_if0);
9763 createRuntimeFunctionCall(Callee: TaskBeginFn, Args: {Ident, ThreadID, TaskData});
9764 CallInst *CI = createRuntimeFunctionCall(Callee: ProxyFn, Args: {ThreadID, TaskData});
9765 CI->setDebugLoc(StaleCI->getDebugLoc());
9766 createRuntimeFunctionCall(Callee: TaskCompleteFn, Args: {Ident, ThreadID, TaskData});
9767 } else if (DepArray) {
9768 // HasNoWait - meaning the task may be deferred. Call
9769 // __kmpc_omp_task_with_deps if there are dependencies,
9770 // else call __kmpc_omp_task
9771 Function *TaskFn =
9772 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_with_deps);
9773 createRuntimeFunctionCall(
9774 Callee: TaskFn,
9775 Args: {Ident, ThreadID, TaskData, NumDeps, DepArray,
9776 ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0),
9777 ConstantPointerNull::get(T: PointerType::getUnqual(C&: M.getContext()))});
9778 } else {
9779 // Emit the @__kmpc_omp_task runtime call to spawn the task
9780 Function *TaskFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task);
9781 createRuntimeFunctionCall(Callee: TaskFn, Args: {Ident, ThreadID, TaskData});
9782 }
9783
9784 StaleCI->eraseFromParent();
9785 for (Instruction *I : llvm::reverse(C&: ToBeDeleted))
9786 I->eraseFromParent();
9787 };
9788 addOutlineInfo(OI: std::move(OI));
9789
9790 LLVM_DEBUG(dbgs() << "Insert block after emitKernelLaunch = \n"
9791 << *(Builder.GetInsertBlock()) << "\n");
9792 LLVM_DEBUG(dbgs() << "Module after emitKernelLaunch = \n"
9793 << *(Builder.GetInsertBlock()->getParent()->getParent())
9794 << "\n");
9795 return Builder.saveIP();
9796}
9797
9798Error OpenMPIRBuilder::emitOffloadingArraysAndArgs(
9799 InsertPointTy AllocaIP, InsertPointTy CodeGenIP, TargetDataInfo &Info,
9800 TargetDataRTArgs &RTArgs, MapInfosTy &CombinedInfo,
9801 CustomMapperCallbackTy CustomMapperCB, bool IsNonContiguous,
9802 bool ForEndCall, function_ref<void(unsigned int, Value *)> DeviceAddrCB) {
9803 if (Error Err =
9804 emitOffloadingArrays(AllocaIP, CodeGenIP, CombinedInfo, Info,
9805 CustomMapperCB, IsNonContiguous, DeviceAddrCB))
9806 return Err;
9807 emitOffloadingArraysArgument(Builder, RTArgs, Info, ForEndCall);
9808 return Error::success();
9809}
9810
9811static void emitTargetCall(
9812 OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder,
9813 OpenMPIRBuilder::InsertPointTy AllocaIP,
9814 ArrayRef<BasicBlock *> DeallocBlocks, OpenMPIRBuilder::TargetDataInfo &Info,
9815 const OpenMPIRBuilder::TargetKernelDefaultAttrs &DefaultAttrs,
9816 const OpenMPIRBuilder::TargetKernelRuntimeAttrs &RuntimeAttrs,
9817 Value *IfCond, Function *OutlinedFn, Constant *OutlinedFnID,
9818 SmallVectorImpl<Value *> &Args,
9819 OpenMPIRBuilder::GenMapInfoCallbackTy GenMapInfoCB,
9820 OpenMPIRBuilder::CustomMapperCallbackTy CustomMapperCB,
9821 const OpenMPIRBuilder::DependenciesInfo &Dependencies, bool HasNoWait,
9822 Value *DynCGroupMem, OMPDynGroupprivateFallbackType DynCGroupMemFallback) {
9823 // Generate a function call to the host fallback implementation of the target
9824 // region. This is called by the host when no offload entry was generated for
9825 // the target region and when the offloading call fails at runtime.
9826 auto &&EmitTargetCallFallbackCB = [&](OpenMPIRBuilder::InsertPointTy IP)
9827 -> OpenMPIRBuilder::InsertPointOrErrorTy {
9828 Builder.restoreIP(IP);
9829 // Ensure the host fallback has the same dyn_ptr ABI as the device.
9830 SmallVector<Value *> FallbackArgs(Args.begin(), Args.end());
9831 FallbackArgs.push_back(
9832 Elt: Constant::getNullValue(Ty: PointerType::getUnqual(C&: Builder.getContext())));
9833 OMPBuilder.createRuntimeFunctionCall(Callee: OutlinedFn, Args: FallbackArgs);
9834 return Builder.saveIP();
9835 };
9836
9837 bool HasDependencies = !Dependencies.empty();
9838 bool RequiresOuterTargetTask = HasNoWait || HasDependencies;
9839
9840 OpenMPIRBuilder::TargetKernelArgs KArgs;
9841
9842 auto TaskBodyCB =
9843 [&](Value *DeviceID, Value *RTLoc,
9844 IRBuilderBase::InsertPoint TargetTaskAllocaIP) -> Error {
9845 // Assume no error was returned because EmitTargetCallFallbackCB doesn't
9846 // produce any.
9847 llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(ValOrErr: [&]() {
9848 // emitKernelLaunch makes the necessary runtime call to offload the
9849 // kernel. We then outline all that code into a separate function
9850 // ('kernel_launch_function' in the pseudo code above). This function is
9851 // then called by the target task proxy function (see
9852 // '@.omp_target_task_proxy_func' in the pseudo code above)
9853 // "@.omp_target_task_proxy_func' is generated by
9854 // emitTargetTaskProxyFunction.
9855 if (OutlinedFnID && DeviceID)
9856 return OMPBuilder.emitKernelLaunch(Loc: Builder, OutlinedFnID,
9857 EmitTargetCallFallbackCB, Args&: KArgs,
9858 DeviceID, RTLoc, AllocaIP: TargetTaskAllocaIP);
9859
9860 // We only need to do the outlining if `DeviceID` is set to avoid calling
9861 // `emitKernelLaunch` if we want to code-gen for the host; e.g. if we are
9862 // generating the `else` branch of an `if` clause.
9863 //
9864 // When OutlinedFnID is set to nullptr, then it's not an offloading call.
9865 // In this case, we execute the host implementation directly.
9866 return EmitTargetCallFallbackCB(OMPBuilder.Builder.saveIP());
9867 }());
9868
9869 OMPBuilder.Builder.restoreIP(IP: AfterIP);
9870 return Error::success();
9871 };
9872
9873 auto &&EmitTargetCallElse =
9874 [&](OpenMPIRBuilder::InsertPointTy AllocaIP,
9875 OpenMPIRBuilder::InsertPointTy CodeGenIP,
9876 ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
9877 // Assume no error was returned because EmitTargetCallFallbackCB doesn't
9878 // produce any.
9879 OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(ValOrErr: [&]() {
9880 if (RequiresOuterTargetTask) {
9881 // Arguments that are intended to be directly forwarded to an
9882 // emitKernelLaunch call are pased as nullptr, since
9883 // OutlinedFnID=nullptr results in that call not being done.
9884 OpenMPIRBuilder::TargetDataRTArgs EmptyRTArgs;
9885 return OMPBuilder.emitTargetTask(TaskBodyCB, /*DeviceID=*/nullptr,
9886 /*RTLoc=*/nullptr, AllocaIP,
9887 Dependencies, RTArgs: EmptyRTArgs, HasNoWait);
9888 }
9889 return EmitTargetCallFallbackCB(Builder.saveIP());
9890 }());
9891
9892 Builder.restoreIP(IP: AfterIP);
9893 return Error::success();
9894 };
9895
9896 auto &&EmitTargetCallThen =
9897 [&](OpenMPIRBuilder::InsertPointTy AllocaIP,
9898 OpenMPIRBuilder::InsertPointTy CodeGenIP,
9899 ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
9900 Info.HasNoWait = HasNoWait;
9901 OpenMPIRBuilder::MapInfosTy &MapInfo = GenMapInfoCB(Builder.saveIP());
9902
9903 OpenMPIRBuilder::TargetDataRTArgs RTArgs;
9904 if (Error Err = OMPBuilder.emitOffloadingArraysAndArgs(
9905 AllocaIP, CodeGenIP: Builder.saveIP(), Info, RTArgs, CombinedInfo&: MapInfo, CustomMapperCB,
9906 /*IsNonContiguous=*/true,
9907 /*ForEndCall=*/false))
9908 return Err;
9909
9910 SmallVector<Value *, 3> NumTeamsC;
9911 for (auto [DefaultVal, RuntimeVal] :
9912 zip_equal(t: DefaultAttrs.MaxTeams, u: RuntimeAttrs.MaxTeams))
9913 NumTeamsC.push_back(Elt: RuntimeVal ? RuntimeVal
9914 : Builder.getInt32(C: DefaultVal));
9915
9916 // Calculate number of threads: 0 if no clauses specified, otherwise it is
9917 // the minimum between optional THREAD_LIMIT and NUM_THREADS clauses.
9918 auto InitMaxThreadsClause = [&Builder](Value *Clause) {
9919 if (Clause)
9920 Clause = Builder.CreateIntCast(V: Clause, DestTy: Builder.getInt32Ty(),
9921 /*isSigned=*/false);
9922 return Clause;
9923 };
9924 auto CombineMaxThreadsClauses = [&Builder](Value *Clause, Value *&Result) {
9925 if (Clause)
9926 Result =
9927 Result ? Builder.CreateSelect(C: Builder.CreateICmpULT(LHS: Result, RHS: Clause),
9928 True: Result, False: Clause)
9929 : Clause;
9930 };
9931
9932 // If a multi-dimensional THREAD_LIMIT is set, it is the OMPX_BARE case, so
9933 // the NUM_THREADS clause is overriden by THREAD_LIMIT.
9934 SmallVector<Value *, 3> NumThreadsC;
9935 Value *MaxThreadsClause =
9936 RuntimeAttrs.TeamsThreadLimit.size() == 1
9937 ? InitMaxThreadsClause(RuntimeAttrs.MaxThreads)
9938 : nullptr;
9939
9940 for (auto [TeamsVal, TargetVal] : zip_equal(
9941 t: RuntimeAttrs.TeamsThreadLimit, u: RuntimeAttrs.TargetThreadLimit)) {
9942 Value *TeamsThreadLimitClause = InitMaxThreadsClause(TeamsVal);
9943 Value *NumThreads = InitMaxThreadsClause(TargetVal);
9944
9945 CombineMaxThreadsClauses(TeamsThreadLimitClause, NumThreads);
9946 CombineMaxThreadsClauses(MaxThreadsClause, NumThreads);
9947
9948 NumThreadsC.push_back(Elt: NumThreads ? NumThreads : Builder.getInt32(C: 0));
9949 }
9950
9951 unsigned NumTargetItems = Info.NumberOfPtrs;
9952 uint32_t SrcLocStrSize;
9953 Constant *SrcLocStr = OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
9954 Value *RTLoc = OMPBuilder.getOrCreateIdent(SrcLocStr, SrcLocStrSize,
9955 LocFlags: llvm::omp::IdentFlag(0), Reserve2Flags: 0);
9956
9957 Value *TripCount = RuntimeAttrs.LoopTripCount
9958 ? Builder.CreateIntCast(V: RuntimeAttrs.LoopTripCount,
9959 DestTy: Builder.getInt64Ty(),
9960 /*isSigned=*/false)
9961 : Builder.getInt64(C: 0);
9962
9963 // Request zero groupprivate bytes by default.
9964 if (!DynCGroupMem)
9965 DynCGroupMem = Builder.getInt32(C: 0);
9966
9967 KArgs = OpenMPIRBuilder::TargetKernelArgs(
9968 NumTargetItems, RTArgs, TripCount, NumTeamsC, NumThreadsC, DynCGroupMem,
9969 HasNoWait, /*StrictBlocksAndThreads=*/false, DynCGroupMemFallback);
9970
9971 // Assume no error was returned because TaskBodyCB and
9972 // EmitTargetCallFallbackCB don't produce any.
9973 OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(ValOrErr: [&]() {
9974 // The presence of certain clauses on the target directive require the
9975 // explicit generation of the target task.
9976 if (RequiresOuterTargetTask)
9977 return OMPBuilder.emitTargetTask(TaskBodyCB, DeviceID: RuntimeAttrs.DeviceID,
9978 RTLoc, AllocaIP, Dependencies,
9979 RTArgs: KArgs.RTArgs, HasNoWait: Info.HasNoWait);
9980
9981 return OMPBuilder.emitKernelLaunch(
9982 Loc: Builder, OutlinedFnID, EmitTargetCallFallbackCB, Args&: KArgs,
9983 DeviceID: RuntimeAttrs.DeviceID, RTLoc, AllocaIP);
9984 }());
9985
9986 Builder.restoreIP(IP: AfterIP);
9987 return Error::success();
9988 };
9989
9990 // If we don't have an ID for the target region, it means an offload entry
9991 // wasn't created. In this case we just run the host fallback directly and
9992 // ignore any potential 'if' clauses.
9993 if (!OutlinedFnID) {
9994 cantFail(Err: EmitTargetCallElse(AllocaIP, Builder.saveIP(), DeallocBlocks));
9995 return;
9996 }
9997
9998 // If there's no 'if' clause, only generate the kernel launch code path.
9999 if (!IfCond) {
10000 cantFail(Err: EmitTargetCallThen(AllocaIP, Builder.saveIP(), DeallocBlocks));
10001 return;
10002 }
10003
10004 cantFail(Err: OMPBuilder.emitIfClause(Cond: IfCond, ThenGen: EmitTargetCallThen,
10005 ElseGen: EmitTargetCallElse, AllocaIP));
10006}
10007
10008OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTarget(
10009 const LocationDescription &Loc, bool IsOffloadEntry, InsertPointTy AllocaIP,
10010 InsertPointTy CodeGenIP, ArrayRef<BasicBlock *> DeallocBlocks,
10011 TargetDataInfo &Info, TargetRegionEntryInfo &EntryInfo,
10012 const TargetKernelDefaultAttrs &DefaultAttrs,
10013 const TargetKernelRuntimeAttrs &RuntimeAttrs, Value *IfCond,
10014 SmallVectorImpl<Value *> &Inputs, GenMapInfoCallbackTy GenMapInfoCB,
10015 OpenMPIRBuilder::TargetBodyGenCallbackTy CBFunc,
10016 OpenMPIRBuilder::TargetGenArgAccessorsCallbackTy ArgAccessorFuncCB,
10017 CustomMapperCallbackTy CustomMapperCB, const DependenciesInfo &Dependencies,
10018 bool HasNowait, Value *DynCGroupMem,
10019 OMPDynGroupprivateFallbackType DynCGroupMemFallback) {
10020
10021 if (!updateToLocation(Loc))
10022 return InsertPointTy();
10023
10024 Builder.restoreIP(IP: CodeGenIP);
10025
10026 Function *OutlinedFn;
10027 Constant *OutlinedFnID = nullptr;
10028 // The target region is outlined into its own function. The LLVM IR for
10029 // the target region itself is generated using the callbacks CBFunc
10030 // and ArgAccessorFuncCB
10031 if (Error Err = emitTargetOutlinedFunction(
10032 OMPBuilder&: *this, Builder, IsOffloadEntry, EntryInfo, DefaultAttrs, OutlinedFn,
10033 OutlinedFnID, Inputs, CBFunc, ArgAccessorFuncCB))
10034 return Err;
10035
10036 // If we are not on the target device, then we need to generate code
10037 // to make a remote call (offload) to the previously outlined function
10038 // that represents the target region. Do that now.
10039 if (!Config.isTargetDevice())
10040 emitTargetCall(OMPBuilder&: *this, Builder, AllocaIP, DeallocBlocks, Info, DefaultAttrs,
10041 RuntimeAttrs, IfCond, OutlinedFn, OutlinedFnID, Args&: Inputs,
10042 GenMapInfoCB, CustomMapperCB, Dependencies, HasNoWait: HasNowait,
10043 DynCGroupMem, DynCGroupMemFallback);
10044 return Builder.saveIP();
10045}
10046
10047std::string OpenMPIRBuilder::getNameWithSeparators(ArrayRef<StringRef> Parts,
10048 StringRef FirstSeparator,
10049 StringRef Separator) {
10050 SmallString<128> Buffer;
10051 llvm::raw_svector_ostream OS(Buffer);
10052 StringRef Sep = FirstSeparator;
10053 for (StringRef Part : Parts) {
10054 OS << Sep << Part;
10055 Sep = Separator;
10056 }
10057 return OS.str().str();
10058}
10059
10060std::string
10061OpenMPIRBuilder::createPlatformSpecificName(ArrayRef<StringRef> Parts) const {
10062 return OpenMPIRBuilder::getNameWithSeparators(Parts, FirstSeparator: Config.firstSeparator(),
10063 Separator: Config.separator());
10064}
10065
10066GlobalVariable *OpenMPIRBuilder::getOrCreateInternalVariable(
10067 Type *Ty, const StringRef &Name, std::optional<unsigned> AddressSpace) {
10068 auto &Elem = *InternalVars.try_emplace(Key: Name, Args: nullptr).first;
10069 if (Elem.second) {
10070 assert(Elem.second->getValueType() == Ty &&
10071 "OMP internal variable has different type than requested");
10072 } else {
10073 // TODO: investigate the appropriate linkage type used for the global
10074 // variable for possibly changing that to internal or private, or maybe
10075 // create different versions of the function for different OMP internal
10076 // variables.
10077 const DataLayout &DL = M.getDataLayout();
10078 // TODO: Investigate why AMDGPU expects AS 0 for globals even though the
10079 // default global AS is 1.
10080 // See double-target-call-with-declare-target.f90 and
10081 // declare-target-vars-in-target-region.f90 libomptarget
10082 // tests.
10083 unsigned AddressSpaceVal = AddressSpace ? *AddressSpace
10084 : M.getTargetTriple().isAMDGPU()
10085 ? 0
10086 : DL.getDefaultGlobalsAddressSpace();
10087 auto Linkage = this->M.getTargetTriple().getArch() == Triple::wasm32
10088 ? GlobalValue::InternalLinkage
10089 : GlobalValue::CommonLinkage;
10090 auto *GV = new GlobalVariable(M, Ty, /*IsConstant=*/false, Linkage,
10091 Constant::getNullValue(Ty), Elem.first(),
10092 /*InsertBefore=*/nullptr,
10093 GlobalValue::NotThreadLocal, AddressSpaceVal);
10094 const llvm::Align TypeAlign = DL.getABITypeAlign(Ty);
10095 const llvm::Align PtrAlign = DL.getPointerABIAlignment(AS: AddressSpaceVal);
10096 GV->setAlignment(std::max(a: TypeAlign, b: PtrAlign));
10097 Elem.second = GV;
10098 }
10099
10100 return Elem.second;
10101}
10102
10103Value *OpenMPIRBuilder::getOMPCriticalRegionLock(StringRef CriticalName) {
10104 std::string Prefix = Twine("gomp_critical_user_", CriticalName).str();
10105 std::string Name = getNameWithSeparators(Parts: {Prefix, "var"}, FirstSeparator: ".", Separator: ".");
10106 return getOrCreateInternalVariable(Ty: KmpCriticalNameTy, Name);
10107}
10108
10109Value *OpenMPIRBuilder::getSizeInBytes(Value *BasePtr) {
10110 LLVMContext &Ctx = Builder.getContext();
10111 Value *Null =
10112 Constant::getNullValue(Ty: PointerType::getUnqual(C&: BasePtr->getContext()));
10113 Value *SizeGep =
10114 Builder.CreateGEP(Ty: BasePtr->getType(), Ptr: Null, IdxList: Builder.getInt32(C: 1));
10115 Value *SizePtrToInt = Builder.CreatePtrToInt(V: SizeGep, DestTy: Type::getInt64Ty(C&: Ctx));
10116 return SizePtrToInt;
10117}
10118
10119GlobalVariable *
10120OpenMPIRBuilder::createOffloadMaptypes(SmallVectorImpl<uint64_t> &Mappings,
10121 std::string VarName) {
10122 llvm::Constant *MaptypesArrayInit =
10123 llvm::ConstantDataArray::get(Context&: M.getContext(), Elts&: Mappings);
10124 auto *MaptypesArrayGlobal = new llvm::GlobalVariable(
10125 M, MaptypesArrayInit->getType(),
10126 /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage, MaptypesArrayInit,
10127 VarName);
10128 MaptypesArrayGlobal->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
10129 return MaptypesArrayGlobal;
10130}
10131
10132void OpenMPIRBuilder::createMapperAllocas(const LocationDescription &Loc,
10133 InsertPointTy AllocaIP,
10134 unsigned NumOperands,
10135 struct MapperAllocas &MapperAllocas) {
10136 if (!updateToLocation(Loc))
10137 return;
10138
10139 auto *ArrI8PtrTy = ArrayType::get(ElementType: Int8Ptr, NumElements: NumOperands);
10140 auto *ArrI64Ty = ArrayType::get(ElementType: Int64, NumElements: NumOperands);
10141 Builder.restoreIP(IP: AllocaIP);
10142 AllocaInst *ArgsBase = Builder.CreateAlloca(
10143 Ty: ArrI8PtrTy, /* ArraySize = */ nullptr, Name: ".offload_baseptrs");
10144 AllocaInst *Args = Builder.CreateAlloca(Ty: ArrI8PtrTy, /* ArraySize = */ nullptr,
10145 Name: ".offload_ptrs");
10146 AllocaInst *ArgSizes = Builder.CreateAlloca(
10147 Ty: ArrI64Ty, /* ArraySize = */ nullptr, Name: ".offload_sizes");
10148 updateToLocation(Loc);
10149 MapperAllocas.ArgsBase = ArgsBase;
10150 MapperAllocas.Args = Args;
10151 MapperAllocas.ArgSizes = ArgSizes;
10152}
10153
10154void OpenMPIRBuilder::emitMapperCall(const LocationDescription &Loc,
10155 Function *MapperFunc, Value *SrcLocInfo,
10156 Value *MaptypesArg, Value *MapnamesArg,
10157 struct MapperAllocas &MapperAllocas,
10158 int64_t DeviceID, unsigned NumOperands) {
10159 if (!updateToLocation(Loc))
10160 return;
10161
10162 auto *ArrI8PtrTy = ArrayType::get(ElementType: Int8Ptr, NumElements: NumOperands);
10163 auto *ArrI64Ty = ArrayType::get(ElementType: Int64, NumElements: NumOperands);
10164 Value *ArgsBaseGEP =
10165 Builder.CreateInBoundsGEP(Ty: ArrI8PtrTy, Ptr: MapperAllocas.ArgsBase,
10166 IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: 0)});
10167 Value *ArgsGEP =
10168 Builder.CreateInBoundsGEP(Ty: ArrI8PtrTy, Ptr: MapperAllocas.Args,
10169 IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: 0)});
10170 Value *ArgSizesGEP =
10171 Builder.CreateInBoundsGEP(Ty: ArrI64Ty, Ptr: MapperAllocas.ArgSizes,
10172 IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: 0)});
10173 Value *NullPtr =
10174 Constant::getNullValue(Ty: PointerType::getUnqual(C&: Int8Ptr->getContext()));
10175 createRuntimeFunctionCall(Callee: MapperFunc, Args: {SrcLocInfo, Builder.getInt64(C: DeviceID),
10176 Builder.getInt32(C: NumOperands),
10177 ArgsBaseGEP, ArgsGEP, ArgSizesGEP,
10178 MaptypesArg, MapnamesArg, NullPtr});
10179}
10180
10181void OpenMPIRBuilder::emitOffloadingArraysArgument(IRBuilderBase &Builder,
10182 TargetDataRTArgs &RTArgs,
10183 TargetDataInfo &Info,
10184 bool ForEndCall) {
10185 assert((!ForEndCall || Info.separateBeginEndCalls()) &&
10186 "expected region end call to runtime only when end call is separate");
10187 auto UnqualPtrTy = PointerType::getUnqual(C&: M.getContext());
10188 auto VoidPtrTy = UnqualPtrTy;
10189 auto VoidPtrPtrTy = UnqualPtrTy;
10190 auto Int64Ty = Type::getInt64Ty(C&: M.getContext());
10191 auto Int64PtrTy = UnqualPtrTy;
10192
10193 if (!Info.NumberOfPtrs) {
10194 RTArgs.BasePointersArray = ConstantPointerNull::get(T: VoidPtrPtrTy);
10195 RTArgs.PointersArray = ConstantPointerNull::get(T: VoidPtrPtrTy);
10196 RTArgs.SizesArray = ConstantPointerNull::get(T: Int64PtrTy);
10197 RTArgs.MapTypesArray = ConstantPointerNull::get(T: Int64PtrTy);
10198 RTArgs.MapNamesArray = ConstantPointerNull::get(T: VoidPtrPtrTy);
10199 RTArgs.MappersArray = ConstantPointerNull::get(T: VoidPtrPtrTy);
10200 return;
10201 }
10202
10203 RTArgs.BasePointersArray = Builder.CreateConstInBoundsGEP2_32(
10204 Ty: ArrayType::get(ElementType: VoidPtrTy, NumElements: Info.NumberOfPtrs),
10205 Ptr: Info.RTArgs.BasePointersArray,
10206 /*Idx0=*/0, /*Idx1=*/0);
10207 RTArgs.PointersArray = Builder.CreateConstInBoundsGEP2_32(
10208 Ty: ArrayType::get(ElementType: VoidPtrTy, NumElements: Info.NumberOfPtrs), Ptr: Info.RTArgs.PointersArray,
10209 /*Idx0=*/0,
10210 /*Idx1=*/0);
10211 RTArgs.SizesArray = Builder.CreateConstInBoundsGEP2_32(
10212 Ty: ArrayType::get(ElementType: Int64Ty, NumElements: Info.NumberOfPtrs), Ptr: Info.RTArgs.SizesArray,
10213 /*Idx0=*/0, /*Idx1=*/0);
10214 RTArgs.MapTypesArray = Builder.CreateConstInBoundsGEP2_32(
10215 Ty: ArrayType::get(ElementType: Int64Ty, NumElements: Info.NumberOfPtrs),
10216 Ptr: ForEndCall && Info.RTArgs.MapTypesArrayEnd ? Info.RTArgs.MapTypesArrayEnd
10217 : Info.RTArgs.MapTypesArray,
10218 /*Idx0=*/0,
10219 /*Idx1=*/0);
10220
10221 // Only emit the mapper information arrays if debug information is
10222 // requested.
10223 if (!Info.EmitDebug)
10224 RTArgs.MapNamesArray = ConstantPointerNull::get(T: VoidPtrPtrTy);
10225 else
10226 RTArgs.MapNamesArray = Builder.CreateConstInBoundsGEP2_32(
10227 Ty: ArrayType::get(ElementType: VoidPtrTy, NumElements: Info.NumberOfPtrs), Ptr: Info.RTArgs.MapNamesArray,
10228 /*Idx0=*/0,
10229 /*Idx1=*/0);
10230 // If there is no user-defined mapper, set the mapper array to nullptr to
10231 // avoid an unnecessary data privatization
10232 if (!Info.HasMapper)
10233 RTArgs.MappersArray = ConstantPointerNull::get(T: VoidPtrPtrTy);
10234 else
10235 RTArgs.MappersArray =
10236 Builder.CreatePointerCast(V: Info.RTArgs.MappersArray, DestTy: VoidPtrPtrTy);
10237}
10238
10239void OpenMPIRBuilder::emitNonContiguousDescriptor(InsertPointTy AllocaIP,
10240 InsertPointTy CodeGenIP,
10241 MapInfosTy &CombinedInfo,
10242 TargetDataInfo &Info) {
10243 MapInfosTy::StructNonContiguousInfo &NonContigInfo =
10244 CombinedInfo.NonContigInfo;
10245
10246 // Build an array of struct descriptor_dim and then assign it to
10247 // offload_args.
10248 //
10249 // struct descriptor_dim {
10250 // uint64_t offset;
10251 // uint64_t count;
10252 // uint64_t stride
10253 // };
10254 Type *Int64Ty = Builder.getInt64Ty();
10255 StructType *DimTy = StructType::create(
10256 Context&: M.getContext(), Elements: ArrayRef<Type *>({Int64Ty, Int64Ty, Int64Ty}),
10257 Name: "struct.descriptor_dim");
10258
10259 enum { OffsetFD = 0, CountFD, StrideFD };
10260 // We need two index variable here since the size of "Dims" is the same as
10261 // the size of Components, however, the size of offset, count, and stride is
10262 // equal to the size of base declaration that is non-contiguous.
10263 for (unsigned I = 0, L = 0, E = NonContigInfo.Dims.size(); I < E; ++I) {
10264 // Skip emitting ir if dimension size is 1 since it cannot be
10265 // non-contiguous.
10266 if (NonContigInfo.Dims[I] == 1)
10267 continue;
10268 Builder.restoreIP(IP: AllocaIP);
10269 ArrayType *ArrayTy = ArrayType::get(ElementType: DimTy, NumElements: NonContigInfo.Dims[I]);
10270 AllocaInst *DimsAddr =
10271 Builder.CreateAlloca(Ty: ArrayTy, /* ArraySize = */ nullptr, Name: "dims");
10272 Builder.restoreIP(IP: CodeGenIP);
10273 for (unsigned II = 0, EE = NonContigInfo.Dims[I]; II < EE; ++II) {
10274 unsigned RevIdx = EE - II - 1;
10275 Value *DimsLVal = Builder.CreateInBoundsGEP(
10276 Ty: ArrayTy, Ptr: DimsAddr, IdxList: {Builder.getInt64(C: 0), Builder.getInt64(C: II)});
10277 // Offset
10278 Value *OffsetLVal = Builder.CreateStructGEP(Ty: DimTy, Ptr: DimsLVal, Idx: OffsetFD);
10279 Builder.CreateAlignedStore(
10280 Val: NonContigInfo.Offsets[L][RevIdx], Ptr: OffsetLVal,
10281 Align: M.getDataLayout().getPrefTypeAlign(Ty: OffsetLVal->getType()));
10282 // Count
10283 Value *CountLVal = Builder.CreateStructGEP(Ty: DimTy, Ptr: DimsLVal, Idx: CountFD);
10284 Builder.CreateAlignedStore(
10285 Val: NonContigInfo.Counts[L][RevIdx], Ptr: CountLVal,
10286 Align: M.getDataLayout().getPrefTypeAlign(Ty: CountLVal->getType()));
10287 // Stride
10288 Value *StrideLVal = Builder.CreateStructGEP(Ty: DimTy, Ptr: DimsLVal, Idx: StrideFD);
10289 Builder.CreateAlignedStore(
10290 Val: NonContigInfo.Strides[L][RevIdx], Ptr: StrideLVal,
10291 Align: M.getDataLayout().getPrefTypeAlign(Ty: CountLVal->getType()));
10292 }
10293 // args[I] = &dims
10294 Builder.restoreIP(IP: CodeGenIP);
10295 Value *DAddr = Builder.CreatePointerBitCastOrAddrSpaceCast(
10296 V: DimsAddr, DestTy: Builder.getPtrTy());
10297 Value *P = Builder.CreateConstInBoundsGEP2_32(
10298 Ty: ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: Info.NumberOfPtrs),
10299 Ptr: Info.RTArgs.PointersArray, Idx0: 0, Idx1: I);
10300 Builder.CreateAlignedStore(
10301 Val: DAddr, Ptr: P, Align: M.getDataLayout().getPrefTypeAlign(Ty: Builder.getPtrTy()));
10302 ++L;
10303 }
10304}
10305
10306void OpenMPIRBuilder::emitUDMapperArrayInitOrDel(
10307 Function *MapperFn, Value *MapperHandle, Value *Base, Value *Begin,
10308 Value *Size, Value *MapType, Value *MapName, TypeSize ElementSize,
10309 BasicBlock *ExitBB, bool IsInit) {
10310 StringRef Prefix = IsInit ? ".init" : ".del";
10311
10312 // Evaluate if this is an array section.
10313 BasicBlock *BodyBB = BasicBlock::Create(
10314 Context&: M.getContext(), Name: createPlatformSpecificName(Parts: {"omp.array", Prefix}));
10315 Value *IsArray =
10316 Builder.CreateICmpSGT(LHS: Size, RHS: Builder.getInt64(C: 1), Name: "omp.arrayinit.isarray");
10317 Value *DeleteBit = Builder.CreateAnd(
10318 LHS: MapType,
10319 RHS: Builder.getInt64(
10320 C: static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10321 OpenMPOffloadMappingFlags::OMP_MAP_DELETE)));
10322 Value *DeleteCond;
10323 Value *Cond;
10324 if (IsInit) {
10325 // base != begin?
10326 Value *BaseIsBegin = Builder.CreateICmpNE(LHS: Base, RHS: Begin);
10327 Cond = Builder.CreateOr(LHS: IsArray, RHS: BaseIsBegin);
10328 DeleteCond = Builder.CreateIsNull(
10329 Arg: DeleteBit,
10330 Name: createPlatformSpecificName(Parts: {"omp.array", Prefix, ".delete"}));
10331 } else {
10332 Cond = IsArray;
10333 DeleteCond = Builder.CreateIsNotNull(
10334 Arg: DeleteBit,
10335 Name: createPlatformSpecificName(Parts: {"omp.array", Prefix, ".delete"}));
10336 }
10337 Cond = Builder.CreateAnd(LHS: Cond, RHS: DeleteCond);
10338 Builder.CreateCondBr(Cond, True: BodyBB, False: ExitBB);
10339
10340 emitBlock(BB: BodyBB, CurFn: MapperFn);
10341 // Get the array size by multiplying element size and element number (i.e., \p
10342 // Size).
10343 Value *ArraySize = Builder.CreateNUWMul(LHS: Size, RHS: Builder.getInt64(C: ElementSize));
10344 // Remove OMP_MAP_TO and OMP_MAP_FROM from the map type, so that it achieves
10345 // memory allocation/deletion purpose only.
10346 Value *MapTypeArg = Builder.CreateAnd(
10347 LHS: MapType,
10348 RHS: Builder.getInt64(
10349 C: ~static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10350 OpenMPOffloadMappingFlags::OMP_MAP_TO |
10351 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
10352 MapTypeArg = Builder.CreateOr(
10353 LHS: MapTypeArg,
10354 RHS: Builder.getInt64(
10355 C: static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10356 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT)));
10357
10358 // Call the runtime API __tgt_push_mapper_component to fill up the runtime
10359 // data structure.
10360 Value *OffloadingArgs[] = {MapperHandle, Base, Begin,
10361 ArraySize, MapTypeArg, MapName};
10362 createRuntimeFunctionCall(
10363 Callee: getOrCreateRuntimeFunction(M, FnID: OMPRTL___tgt_push_mapper_component),
10364 Args: OffloadingArgs);
10365}
10366
10367Expected<Function *> OpenMPIRBuilder::emitUserDefinedMapper(
10368 function_ref<MapInfosOrErrorTy(InsertPointTy CodeGenIP, llvm::Value *PtrPHI,
10369 llvm::Value *BeginArg)>
10370 GenMapInfoCB,
10371 Type *ElemTy, StringRef FuncName, CustomMapperCallbackTy CustomMapperCB,
10372 bool PreserveMemberOfFlags) {
10373 SmallVector<Type *> Params;
10374 Params.emplace_back(Args: Builder.getPtrTy());
10375 Params.emplace_back(Args: Builder.getPtrTy());
10376 Params.emplace_back(Args: Builder.getPtrTy());
10377 Params.emplace_back(Args: Builder.getInt64Ty());
10378 Params.emplace_back(Args: Builder.getInt64Ty());
10379 Params.emplace_back(Args: Builder.getPtrTy());
10380
10381 auto *FnTy =
10382 FunctionType::get(Result: Builder.getVoidTy(), Params, /* IsVarArg */ isVarArg: false);
10383
10384 SmallString<64> TyStr;
10385 raw_svector_ostream Out(TyStr);
10386 Function *MapperFn =
10387 Function::Create(Ty: FnTy, Linkage: GlobalValue::InternalLinkage, N: FuncName, M);
10388 MapperFn->addFnAttr(Kind: Attribute::NoInline);
10389 MapperFn->addFnAttr(Kind: Attribute::NoUnwind);
10390 MapperFn->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
10391 MapperFn->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
10392 MapperFn->addParamAttr(ArgNo: 2, Kind: Attribute::NoUndef);
10393 MapperFn->addParamAttr(ArgNo: 3, Kind: Attribute::NoUndef);
10394 MapperFn->addParamAttr(ArgNo: 4, Kind: Attribute::NoUndef);
10395 MapperFn->addParamAttr(ArgNo: 5, Kind: Attribute::NoUndef);
10396
10397 // Start the mapper function code generation.
10398 BasicBlock *EntryBB = BasicBlock::Create(Context&: M.getContext(), Name: "entry", Parent: MapperFn);
10399 auto SavedIP = Builder.saveIP();
10400 Builder.SetInsertPoint(EntryBB);
10401
10402 Value *MapperHandle = MapperFn->getArg(i: 0);
10403 Value *BaseIn = MapperFn->getArg(i: 1);
10404 Value *BeginIn = MapperFn->getArg(i: 2);
10405 Value *Size = MapperFn->getArg(i: 3);
10406 Value *MapType = MapperFn->getArg(i: 4);
10407 Value *MapName = MapperFn->getArg(i: 5);
10408
10409 // Compute the starting and end addresses of array elements.
10410 // Prepare common arguments for array initiation and deletion.
10411 // Convert the size in bytes into the number of array elements.
10412 TypeSize ElementSize = M.getDataLayout().getTypeStoreSize(Ty: ElemTy);
10413 Size = Builder.CreateExactUDiv(LHS: Size, RHS: Builder.getInt64(C: ElementSize));
10414 Value *PtrBegin = BeginIn;
10415 Value *PtrEnd = Builder.CreateGEP(Ty: ElemTy, Ptr: PtrBegin, IdxList: Size);
10416
10417 // Emit array initiation if this is an array section and \p MapType indicates
10418 // that memory allocation is required.
10419 BasicBlock *HeadBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp.arraymap.head");
10420 emitUDMapperArrayInitOrDel(MapperFn, MapperHandle, Base: BaseIn, Begin: BeginIn, Size,
10421 MapType, MapName, ElementSize, ExitBB: HeadBB,
10422 /*IsInit=*/true);
10423
10424 // Emit a for loop to iterate through SizeArg of elements and map all of them.
10425
10426 // Emit the loop header block.
10427 emitBlock(BB: HeadBB, CurFn: MapperFn);
10428 BasicBlock *BodyBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp.arraymap.body");
10429 BasicBlock *DoneBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp.done");
10430 // Evaluate whether the initial condition is satisfied.
10431 Value *IsEmpty =
10432 Builder.CreateICmpEQ(LHS: PtrBegin, RHS: PtrEnd, Name: "omp.arraymap.isempty");
10433 Builder.CreateCondBr(Cond: IsEmpty, True: DoneBB, False: BodyBB);
10434
10435 // Emit the loop body block.
10436 emitBlock(BB: BodyBB, CurFn: MapperFn);
10437 BasicBlock *LastBB = BodyBB;
10438 PHINode *PtrPHI =
10439 Builder.CreatePHI(Ty: PtrBegin->getType(), NumReservedValues: 2, Name: "omp.arraymap.ptrcurrent");
10440 PtrPHI->addIncoming(V: PtrBegin, BB: HeadBB);
10441
10442 // Get map clause information. Fill up the arrays with all mapped variables.
10443 MapInfosOrErrorTy Info = GenMapInfoCB(Builder.saveIP(), PtrPHI, BeginIn);
10444 if (!Info)
10445 return Info.takeError();
10446
10447 // Call the runtime API __tgt_mapper_num_components to get the number of
10448 // pre-existing components.
10449 Value *OffloadingArgs[] = {MapperHandle};
10450 Value *PreviousSize = createRuntimeFunctionCall(
10451 Callee: getOrCreateRuntimeFunction(M, FnID: OMPRTL___tgt_mapper_num_components),
10452 Args: OffloadingArgs);
10453 Value *ShiftedPreviousSize =
10454 Builder.CreateShl(LHS: PreviousSize, RHS: Builder.getInt64(C: getFlagMemberOffset()));
10455
10456 // Fill up the runtime mapper handle for all components.
10457 for (unsigned I = 0; I < Info->BasePointers.size(); ++I) {
10458 Value *CurBaseArg = Info->BasePointers[I];
10459 Value *CurBeginArg = Info->Pointers[I];
10460 Value *CurSizeArg = Info->Sizes[I];
10461 Value *CurNameArg = Info->Names.size()
10462 ? Info->Names[I]
10463 : Constant::getNullValue(Ty: Builder.getPtrTy());
10464
10465 // Extract the MEMBER_OF field from the map type.
10466 Value *OriMapType = Builder.getInt64(
10467 C: static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10468 Info->Types[I]));
10469 Value *MemberMapType;
10470 if (PreserveMemberOfFlags) {
10471 constexpr uint64_t MemberOfMask =
10472 static_cast<uint64_t>(OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);
10473 uint64_t OrigFlags =
10474 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10475 Info->Types[I]);
10476 bool HasMemberOf = (OrigFlags & MemberOfMask) != 0;
10477 if (HasMemberOf)
10478 MemberMapType = Builder.CreateNUWAdd(LHS: OriMapType, RHS: ShiftedPreviousSize);
10479 else
10480 MemberMapType = OriMapType;
10481 } else {
10482 MemberMapType = Builder.CreateNUWAdd(LHS: OriMapType, RHS: ShiftedPreviousSize);
10483 }
10484
10485 // Combine the map type inherited from user-defined mapper with that
10486 // specified in the program. According to the OMP_MAP_TO and OMP_MAP_FROM
10487 // bits of the \a MapType, which is the input argument of the mapper
10488 // function, the following code will set the OMP_MAP_TO and OMP_MAP_FROM
10489 // bits of MemberMapType.
10490 // [OpenMP 5.0], 1.2.6. map-type decay.
10491 // | alloc | to | from | tofrom | release | delete
10492 // ----------------------------------------------------------
10493 // alloc | alloc | alloc | alloc | alloc | release | delete
10494 // to | alloc | to | alloc | to | release | delete
10495 // from | alloc | alloc | from | from | release | delete
10496 // tofrom | alloc | to | from | tofrom | release | delete
10497 Value *LeftToFrom = Builder.CreateAnd(
10498 LHS: MapType,
10499 RHS: Builder.getInt64(
10500 C: static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10501 OpenMPOffloadMappingFlags::OMP_MAP_TO |
10502 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
10503 BasicBlock *AllocBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp.type.alloc");
10504 BasicBlock *AllocElseBB =
10505 BasicBlock::Create(Context&: M.getContext(), Name: "omp.type.alloc.else");
10506 BasicBlock *ToBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp.type.to");
10507 BasicBlock *ToElseBB =
10508 BasicBlock::Create(Context&: M.getContext(), Name: "omp.type.to.else");
10509 BasicBlock *FromBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp.type.from");
10510 BasicBlock *EndBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp.type.end");
10511 Value *IsAlloc = Builder.CreateIsNull(Arg: LeftToFrom);
10512 Builder.CreateCondBr(Cond: IsAlloc, True: AllocBB, False: AllocElseBB);
10513 // In case of alloc, clear OMP_MAP_TO and OMP_MAP_FROM.
10514 emitBlock(BB: AllocBB, CurFn: MapperFn);
10515 Value *AllocMapType = Builder.CreateAnd(
10516 LHS: MemberMapType,
10517 RHS: Builder.getInt64(
10518 C: ~static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10519 OpenMPOffloadMappingFlags::OMP_MAP_TO |
10520 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
10521 Builder.CreateBr(Dest: EndBB);
10522 emitBlock(BB: AllocElseBB, CurFn: MapperFn);
10523 Value *IsTo = Builder.CreateICmpEQ(
10524 LHS: LeftToFrom,
10525 RHS: Builder.getInt64(
10526 C: static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10527 OpenMPOffloadMappingFlags::OMP_MAP_TO)));
10528 Builder.CreateCondBr(Cond: IsTo, True: ToBB, False: ToElseBB);
10529 // In case of to, clear OMP_MAP_FROM.
10530 emitBlock(BB: ToBB, CurFn: MapperFn);
10531 Value *ToMapType = Builder.CreateAnd(
10532 LHS: MemberMapType,
10533 RHS: Builder.getInt64(
10534 C: ~static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10535 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
10536 Builder.CreateBr(Dest: EndBB);
10537 emitBlock(BB: ToElseBB, CurFn: MapperFn);
10538 Value *IsFrom = Builder.CreateICmpEQ(
10539 LHS: LeftToFrom,
10540 RHS: Builder.getInt64(
10541 C: static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10542 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
10543 Builder.CreateCondBr(Cond: IsFrom, True: FromBB, False: EndBB);
10544 // In case of from, clear OMP_MAP_TO.
10545 emitBlock(BB: FromBB, CurFn: MapperFn);
10546 Value *FromMapType = Builder.CreateAnd(
10547 LHS: MemberMapType,
10548 RHS: Builder.getInt64(
10549 C: ~static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10550 OpenMPOffloadMappingFlags::OMP_MAP_TO)));
10551 // In case of tofrom, do nothing.
10552 emitBlock(BB: EndBB, CurFn: MapperFn);
10553 LastBB = EndBB;
10554 PHINode *CurMapType =
10555 Builder.CreatePHI(Ty: Builder.getInt64Ty(), NumReservedValues: 4, Name: "omp.maptype");
10556 CurMapType->addIncoming(V: AllocMapType, BB: AllocBB);
10557 CurMapType->addIncoming(V: ToMapType, BB: ToBB);
10558 CurMapType->addIncoming(V: FromMapType, BB: FromBB);
10559 CurMapType->addIncoming(V: MemberMapType, BB: ToElseBB);
10560
10561 Value *OffloadingArgs[] = {MapperHandle, CurBaseArg, CurBeginArg,
10562 CurSizeArg, CurMapType, CurNameArg};
10563
10564 auto ChildMapperFn = CustomMapperCB(I);
10565 if (!ChildMapperFn)
10566 return ChildMapperFn.takeError();
10567 if (*ChildMapperFn) {
10568 // Call the corresponding mapper function.
10569 createRuntimeFunctionCall(Callee: *ChildMapperFn, Args: OffloadingArgs)
10570 ->setDoesNotThrow();
10571 } else {
10572 // Call the runtime API __tgt_push_mapper_component to fill up the runtime
10573 // data structure.
10574 createRuntimeFunctionCall(
10575 Callee: getOrCreateRuntimeFunction(M, FnID: OMPRTL___tgt_push_mapper_component),
10576 Args: OffloadingArgs);
10577 }
10578 }
10579
10580 // Update the pointer to point to the next element that needs to be mapped,
10581 // and check whether we have mapped all elements.
10582 Value *PtrNext = Builder.CreateConstGEP1_32(Ty: ElemTy, Ptr: PtrPHI, /*Idx0=*/1,
10583 Name: "omp.arraymap.next");
10584 PtrPHI->addIncoming(V: PtrNext, BB: LastBB);
10585 Value *IsDone = Builder.CreateICmpEQ(LHS: PtrNext, RHS: PtrEnd, Name: "omp.arraymap.isdone");
10586 BasicBlock *ExitBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp.arraymap.exit");
10587 Builder.CreateCondBr(Cond: IsDone, True: ExitBB, False: BodyBB);
10588
10589 emitBlock(BB: ExitBB, CurFn: MapperFn);
10590 // Emit array deletion if this is an array section and \p MapType indicates
10591 // that deletion is required.
10592 emitUDMapperArrayInitOrDel(MapperFn, MapperHandle, Base: BaseIn, Begin: BeginIn, Size,
10593 MapType, MapName, ElementSize, ExitBB: DoneBB,
10594 /*IsInit=*/false);
10595
10596 // Emit the function exit block.
10597 emitBlock(BB: DoneBB, CurFn: MapperFn, /*IsFinished=*/true);
10598
10599 Builder.CreateRetVoid();
10600 Builder.restoreIP(IP: SavedIP);
10601 return MapperFn;
10602}
10603
10604Error OpenMPIRBuilder::emitOffloadingArrays(
10605 InsertPointTy AllocaIP, InsertPointTy CodeGenIP, MapInfosTy &CombinedInfo,
10606 TargetDataInfo &Info, CustomMapperCallbackTy CustomMapperCB,
10607 bool IsNonContiguous,
10608 function_ref<void(unsigned int, Value *)> DeviceAddrCB) {
10609
10610 // Reset the array information.
10611 Info.clearArrayInfo();
10612 Info.NumberOfPtrs = CombinedInfo.BasePointers.size();
10613
10614 if (Info.NumberOfPtrs == 0)
10615 return Error::success();
10616
10617 Builder.restoreIP(IP: AllocaIP);
10618 // Detect if we have any capture size requiring runtime evaluation of the
10619 // size so that a constant array could be eventually used.
10620 ArrayType *PointerArrayType =
10621 ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: Info.NumberOfPtrs);
10622
10623 Info.RTArgs.BasePointersArray = Builder.CreateAlloca(
10624 Ty: PointerArrayType, /* ArraySize = */ nullptr, Name: ".offload_baseptrs");
10625
10626 Info.RTArgs.PointersArray = Builder.CreateAlloca(
10627 Ty: PointerArrayType, /* ArraySize = */ nullptr, Name: ".offload_ptrs");
10628 AllocaInst *MappersArray = Builder.CreateAlloca(
10629 Ty: PointerArrayType, /* ArraySize = */ nullptr, Name: ".offload_mappers");
10630 Info.RTArgs.MappersArray = MappersArray;
10631
10632 // If we don't have any VLA types or other types that require runtime
10633 // evaluation, we can use a constant array for the map sizes, otherwise we
10634 // need to fill up the arrays as we do for the pointers.
10635 Type *Int64Ty = Builder.getInt64Ty();
10636 SmallVector<Constant *> ConstSizes(CombinedInfo.Sizes.size(),
10637 ConstantInt::get(Ty: Int64Ty, V: 0));
10638 SmallBitVector RuntimeSizes(CombinedInfo.Sizes.size());
10639 for (unsigned I = 0, E = CombinedInfo.Sizes.size(); I < E; ++I) {
10640 bool IsNonContigEntry =
10641 IsNonContiguous &&
10642 (static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10643 CombinedInfo.Types[I] &
10644 OpenMPOffloadMappingFlags::OMP_MAP_NON_CONTIG) != 0);
10645 // For NON_CONTIG entries, ArgSizes stores the dimension count (number of
10646 // descriptor_dim records), not the byte size.
10647 if (IsNonContigEntry) {
10648 assert(I < CombinedInfo.NonContigInfo.Dims.size() &&
10649 "Index must be in-bounds for NON_CONTIG Dims array");
10650 const uint64_t DimCount = CombinedInfo.NonContigInfo.Dims[I];
10651 assert(DimCount > 0 && "NON_CONTIG DimCount must be > 0");
10652 ConstSizes[I] = ConstantInt::get(Ty: Int64Ty, V: DimCount);
10653 continue;
10654 }
10655 if (auto *CI = dyn_cast<Constant>(Val: CombinedInfo.Sizes[I])) {
10656 if (!isa<ConstantExpr>(Val: CI) && !isa<GlobalValue>(Val: CI)) {
10657 ConstSizes[I] = CI;
10658 continue;
10659 }
10660 }
10661 RuntimeSizes.set(I);
10662 }
10663
10664 if (RuntimeSizes.all()) {
10665 ArrayType *SizeArrayType = ArrayType::get(ElementType: Int64Ty, NumElements: Info.NumberOfPtrs);
10666 Info.RTArgs.SizesArray = Builder.CreateAlloca(
10667 Ty: SizeArrayType, /* ArraySize = */ nullptr, Name: ".offload_sizes");
10668 restoreIPandDebugLoc(Builder, IP: CodeGenIP);
10669 } else {
10670 auto *SizesArrayInit = ConstantArray::get(
10671 T: ArrayType::get(ElementType: Int64Ty, NumElements: ConstSizes.size()), V: ConstSizes);
10672 std::string Name = createPlatformSpecificName(Parts: {"offload_sizes"});
10673 auto *SizesArrayGbl =
10674 new GlobalVariable(M, SizesArrayInit->getType(), /*isConstant=*/true,
10675 GlobalValue::PrivateLinkage, SizesArrayInit, Name);
10676 SizesArrayGbl->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
10677
10678 if (!RuntimeSizes.any()) {
10679 Info.RTArgs.SizesArray = SizesArrayGbl;
10680 } else {
10681 unsigned IndexSize = M.getDataLayout().getIndexSizeInBits(AS: 0);
10682 Align OffloadSizeAlign = M.getDataLayout().getABIIntegerTypeAlignment(BitWidth: 64);
10683 ArrayType *SizeArrayType = ArrayType::get(ElementType: Int64Ty, NumElements: Info.NumberOfPtrs);
10684 AllocaInst *Buffer = Builder.CreateAlloca(
10685 Ty: SizeArrayType, /* ArraySize = */ nullptr, Name: ".offload_sizes");
10686 Buffer->setAlignment(OffloadSizeAlign);
10687 restoreIPandDebugLoc(Builder, IP: CodeGenIP);
10688 Builder.CreateMemCpy(
10689 Dst: Buffer, DstAlign: M.getDataLayout().getPrefTypeAlign(Ty: Buffer->getType()),
10690 Src: SizesArrayGbl, SrcAlign: OffloadSizeAlign,
10691 Size: Builder.getIntN(
10692 N: IndexSize,
10693 C: Buffer->getAllocationSize(DL: M.getDataLayout())->getFixedValue()));
10694
10695 Info.RTArgs.SizesArray = Buffer;
10696 }
10697 restoreIPandDebugLoc(Builder, IP: CodeGenIP);
10698 }
10699
10700 // The map types are always constant so we don't need to generate code to
10701 // fill arrays. Instead, we create an array constant.
10702 SmallVector<uint64_t, 4> Mapping;
10703 for (auto mapFlag : CombinedInfo.Types)
10704 Mapping.push_back(
10705 Elt: static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10706 mapFlag));
10707 std::string MaptypesName = createPlatformSpecificName(Parts: {"offload_maptypes"});
10708 auto *MapTypesArrayGbl = createOffloadMaptypes(Mappings&: Mapping, VarName: MaptypesName);
10709 Info.RTArgs.MapTypesArray = MapTypesArrayGbl;
10710
10711 // The information types are only built if provided.
10712 if (!CombinedInfo.Names.empty()) {
10713 auto *MapNamesArrayGbl = createOffloadMapnames(
10714 Names&: CombinedInfo.Names, VarName: createPlatformSpecificName(Parts: {"offload_mapnames"}));
10715 Info.RTArgs.MapNamesArray = MapNamesArrayGbl;
10716 Info.EmitDebug = true;
10717 } else {
10718 Info.RTArgs.MapNamesArray =
10719 Constant::getNullValue(Ty: PointerType::getUnqual(C&: Builder.getContext()));
10720 Info.EmitDebug = false;
10721 }
10722
10723 // If there's a present map type modifier, it must not be applied to the end
10724 // of a region, so generate a separate map type array in that case.
10725 if (Info.separateBeginEndCalls()) {
10726 bool EndMapTypesDiffer = false;
10727 for (uint64_t &Type : Mapping) {
10728 if (Type & static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10729 OpenMPOffloadMappingFlags::OMP_MAP_PRESENT)) {
10730 Type &= ~static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10731 OpenMPOffloadMappingFlags::OMP_MAP_PRESENT);
10732 EndMapTypesDiffer = true;
10733 }
10734 }
10735 if (EndMapTypesDiffer) {
10736 MapTypesArrayGbl = createOffloadMaptypes(Mappings&: Mapping, VarName: MaptypesName);
10737 Info.RTArgs.MapTypesArrayEnd = MapTypesArrayGbl;
10738 }
10739 }
10740
10741 PointerType *PtrTy = Builder.getPtrTy();
10742 for (unsigned I = 0; I < Info.NumberOfPtrs; ++I) {
10743 Value *BPVal = CombinedInfo.BasePointers[I];
10744 Value *BP = Builder.CreateConstInBoundsGEP2_32(
10745 Ty: ArrayType::get(ElementType: PtrTy, NumElements: Info.NumberOfPtrs), Ptr: Info.RTArgs.BasePointersArray,
10746 Idx0: 0, Idx1: I);
10747 Builder.CreateAlignedStore(Val: BPVal, Ptr: BP,
10748 Align: M.getDataLayout().getPrefTypeAlign(Ty: PtrTy));
10749
10750 if (Info.requiresDevicePointerInfo()) {
10751 if (CombinedInfo.DevicePointers[I] == DeviceInfoTy::Pointer) {
10752 CodeGenIP = Builder.saveIP();
10753 Builder.restoreIP(IP: AllocaIP);
10754 Info.DevicePtrInfoMap[BPVal] = {BP, Builder.CreateAlloca(Ty: PtrTy)};
10755 Builder.restoreIP(IP: CodeGenIP);
10756 if (DeviceAddrCB)
10757 DeviceAddrCB(I, Info.DevicePtrInfoMap[BPVal].second);
10758 } else if (CombinedInfo.DevicePointers[I] == DeviceInfoTy::Address) {
10759 Info.DevicePtrInfoMap[BPVal] = {BP, BP};
10760 if (DeviceAddrCB)
10761 DeviceAddrCB(I, BP);
10762 }
10763 }
10764
10765 Value *PVal = CombinedInfo.Pointers[I];
10766 Value *P = Builder.CreateConstInBoundsGEP2_32(
10767 Ty: ArrayType::get(ElementType: PtrTy, NumElements: Info.NumberOfPtrs), Ptr: Info.RTArgs.PointersArray, Idx0: 0,
10768 Idx1: I);
10769 // TODO: Check alignment correct.
10770 Builder.CreateAlignedStore(Val: PVal, Ptr: P,
10771 Align: M.getDataLayout().getPrefTypeAlign(Ty: PtrTy));
10772
10773 if (RuntimeSizes.test(Idx: I)) {
10774 Value *S = Builder.CreateConstInBoundsGEP2_32(
10775 Ty: ArrayType::get(ElementType: Int64Ty, NumElements: Info.NumberOfPtrs), Ptr: Info.RTArgs.SizesArray,
10776 /*Idx0=*/0,
10777 /*Idx1=*/I);
10778 Builder.CreateAlignedStore(Val: Builder.CreateIntCast(V: CombinedInfo.Sizes[I],
10779 DestTy: Int64Ty,
10780 /*isSigned=*/true),
10781 Ptr: S, Align: M.getDataLayout().getPrefTypeAlign(Ty: PtrTy));
10782 }
10783 // Fill up the mapper array.
10784 unsigned IndexSize = M.getDataLayout().getIndexSizeInBits(AS: 0);
10785 Value *MFunc = ConstantPointerNull::get(T: PtrTy);
10786
10787 auto CustomMFunc = CustomMapperCB(I);
10788 if (!CustomMFunc)
10789 return CustomMFunc.takeError();
10790 if (*CustomMFunc)
10791 MFunc = Builder.CreatePointerCast(V: *CustomMFunc, DestTy: PtrTy);
10792
10793 Value *MAddr = Builder.CreateInBoundsGEP(
10794 Ty: PointerArrayType, Ptr: MappersArray,
10795 IdxList: {Builder.getIntN(N: IndexSize, C: 0), Builder.getIntN(N: IndexSize, C: I)});
10796 Builder.CreateAlignedStore(
10797 Val: MFunc, Ptr: MAddr, Align: M.getDataLayout().getPrefTypeAlign(Ty: MAddr->getType()));
10798 }
10799
10800 if (!IsNonContiguous || CombinedInfo.NonContigInfo.Offsets.empty() ||
10801 Info.NumberOfPtrs == 0)
10802 return Error::success();
10803 emitNonContiguousDescriptor(AllocaIP, CodeGenIP, CombinedInfo, Info);
10804 return Error::success();
10805}
10806
10807void OpenMPIRBuilder::emitBranch(BasicBlock *Target) {
10808 BasicBlock *CurBB = Builder.GetInsertBlock();
10809
10810 if (!CurBB || CurBB->hasTerminator()) {
10811 // If there is no insert point or the previous block is already
10812 // terminated, don't touch it.
10813 } else {
10814 // Otherwise, create a fall-through branch.
10815 Builder.CreateBr(Dest: Target);
10816 }
10817
10818 Builder.ClearInsertionPoint();
10819}
10820
10821void OpenMPIRBuilder::emitBlock(BasicBlock *BB, Function *CurFn,
10822 bool IsFinished) {
10823 BasicBlock *CurBB = Builder.GetInsertBlock();
10824
10825 // Fall out of the current block (if necessary).
10826 emitBranch(Target: BB);
10827
10828 if (IsFinished && BB->use_empty()) {
10829 BB->eraseFromParent();
10830 return;
10831 }
10832
10833 // Place the block after the current block, if possible, or else at
10834 // the end of the function.
10835 if (CurBB && CurBB->getParent())
10836 CurFn->insert(Position: std::next(x: CurBB->getIterator()), BB);
10837 else
10838 CurFn->insert(Position: CurFn->end(), BB);
10839 Builder.SetInsertPoint(BB);
10840}
10841
10842Error OpenMPIRBuilder::emitIfClause(Value *Cond, BodyGenCallbackTy ThenGen,
10843 BodyGenCallbackTy ElseGen,
10844 InsertPointTy AllocaIP,
10845 ArrayRef<BasicBlock *> DeallocBlocks) {
10846 // If the condition constant folds and can be elided, try to avoid emitting
10847 // the condition and the dead arm of the if/else.
10848 if (auto *CI = dyn_cast<ConstantInt>(Val: Cond)) {
10849 auto CondConstant = CI->getSExtValue();
10850 if (CondConstant)
10851 return ThenGen(AllocaIP, Builder.saveIP(), DeallocBlocks);
10852
10853 return ElseGen(AllocaIP, Builder.saveIP(), DeallocBlocks);
10854 }
10855
10856 Function *CurFn = Builder.GetInsertBlock()->getParent();
10857
10858 // Otherwise, the condition did not fold, or we couldn't elide it. Just
10859 // emit the conditional branch.
10860 BasicBlock *ThenBlock = BasicBlock::Create(Context&: M.getContext(), Name: "omp_if.then");
10861 BasicBlock *ElseBlock = BasicBlock::Create(Context&: M.getContext(), Name: "omp_if.else");
10862 BasicBlock *ContBlock = BasicBlock::Create(Context&: M.getContext(), Name: "omp_if.end");
10863 Builder.CreateCondBr(Cond, True: ThenBlock, False: ElseBlock);
10864 // Emit the 'then' code.
10865 emitBlock(BB: ThenBlock, CurFn);
10866 if (Error Err = ThenGen(AllocaIP, Builder.saveIP(), DeallocBlocks))
10867 return Err;
10868 emitBranch(Target: ContBlock);
10869 // Emit the 'else' code if present.
10870 // There is no need to emit line number for unconditional branch.
10871 emitBlock(BB: ElseBlock, CurFn);
10872 if (Error Err = ElseGen(AllocaIP, Builder.saveIP(), DeallocBlocks))
10873 return Err;
10874 // There is no need to emit line number for unconditional branch.
10875 emitBranch(Target: ContBlock);
10876 // Emit the continuation block for code after the if.
10877 emitBlock(BB: ContBlock, CurFn, /*IsFinished=*/true);
10878 return Error::success();
10879}
10880
10881bool OpenMPIRBuilder::checkAndEmitFlushAfterAtomic(
10882 const LocationDescription &Loc, llvm::AtomicOrdering AO, AtomicKind AK) {
10883 assert(!(AO == AtomicOrdering::NotAtomic ||
10884 AO == llvm::AtomicOrdering::Unordered) &&
10885 "Unexpected Atomic Ordering.");
10886
10887 bool Flush = false;
10888 llvm::AtomicOrdering FlushAO = AtomicOrdering::Monotonic;
10889
10890 switch (AK) {
10891 case Read:
10892 if (AO == AtomicOrdering::Acquire || AO == AtomicOrdering::AcquireRelease ||
10893 AO == AtomicOrdering::SequentiallyConsistent) {
10894 FlushAO = AtomicOrdering::Acquire;
10895 Flush = true;
10896 }
10897 break;
10898 case Write:
10899 case Compare:
10900 case Update:
10901 if (AO == AtomicOrdering::Release || AO == AtomicOrdering::AcquireRelease ||
10902 AO == AtomicOrdering::SequentiallyConsistent) {
10903 FlushAO = AtomicOrdering::Release;
10904 Flush = true;
10905 }
10906 break;
10907 case Capture:
10908 switch (AO) {
10909 case AtomicOrdering::Acquire:
10910 FlushAO = AtomicOrdering::Acquire;
10911 Flush = true;
10912 break;
10913 case AtomicOrdering::Release:
10914 FlushAO = AtomicOrdering::Release;
10915 Flush = true;
10916 break;
10917 case AtomicOrdering::AcquireRelease:
10918 case AtomicOrdering::SequentiallyConsistent:
10919 FlushAO = AtomicOrdering::AcquireRelease;
10920 Flush = true;
10921 break;
10922 default:
10923 // do nothing - leave silently.
10924 break;
10925 }
10926 }
10927
10928 if (Flush) {
10929 // Currently Flush RT call still doesn't take memory_ordering, so for when
10930 // that happens, this tries to do the resolution of which atomic ordering
10931 // to use with but issue the flush call
10932 // TODO: pass `FlushAO` after memory ordering support is added
10933 (void)FlushAO;
10934 emitFlush(Loc);
10935 }
10936
10937 // for AO == AtomicOrdering::Monotonic and all other case combinations
10938 // do nothing
10939 return Flush;
10940}
10941
10942OpenMPIRBuilder::InsertPointTy
10943OpenMPIRBuilder::createAtomicRead(const LocationDescription &Loc,
10944 AtomicOpValue &X, AtomicOpValue &V,
10945 AtomicOrdering AO, InsertPointTy AllocaIP) {
10946 if (!updateToLocation(Loc))
10947 return Loc.IP;
10948
10949 assert(X.Var->getType()->isPointerTy() &&
10950 "OMP Atomic expects a pointer to target memory");
10951 Type *XElemTy = X.ElemTy;
10952 assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||
10953 XElemTy->isPointerTy() || XElemTy->isStructTy()) &&
10954 "OMP atomic read expected a scalar type");
10955
10956 Value *XRead = nullptr;
10957
10958 if (XElemTy->isIntegerTy()) {
10959 LoadInst *XLD =
10960 Builder.CreateLoad(Ty: XElemTy, Ptr: X.Var, isVolatile: X.IsVolatile, Name: "omp.atomic.read");
10961 XLD->setAtomic(Ordering: AO);
10962 XRead = cast<Value>(Val: XLD);
10963 } else if (XElemTy->isStructTy()) {
10964 // FIXME: Add checks to ensure __atomic_load is emitted iff the
10965 // target does not support `atomicrmw` of the size of the struct
10966 LoadInst *OldVal = Builder.CreateLoad(Ty: XElemTy, Ptr: X.Var, Name: "omp.atomic.read");
10967 OldVal->setAtomic(Ordering: AO);
10968 const DataLayout &DL = OldVal->getModule()->getDataLayout();
10969 unsigned LoadSize = DL.getTypeStoreSize(Ty: XElemTy);
10970 OpenMPIRBuilder::AtomicInfo atomicInfo(
10971 &Builder, XElemTy, LoadSize * 8, LoadSize * 8, OldVal->getAlign(),
10972 OldVal->getAlign(), true /* UseLibcall */, AllocaIP, X.Var);
10973 auto AtomicLoadRes = atomicInfo.EmitAtomicLoadLibcall(AO);
10974 XRead = AtomicLoadRes.first;
10975 OldVal->eraseFromParent();
10976 } else {
10977 // We need to perform atomic op as integer
10978 IntegerType *IntCastTy =
10979 IntegerType::get(C&: M.getContext(), NumBits: XElemTy->getScalarSizeInBits());
10980 LoadInst *XLoad =
10981 Builder.CreateLoad(Ty: IntCastTy, Ptr: X.Var, isVolatile: X.IsVolatile, Name: "omp.atomic.load");
10982 XLoad->setAtomic(Ordering: AO);
10983 if (XElemTy->isFloatingPointTy()) {
10984 XRead = Builder.CreateBitCast(V: XLoad, DestTy: XElemTy, Name: "atomic.flt.cast");
10985 } else {
10986 XRead = Builder.CreateIntToPtr(V: XLoad, DestTy: XElemTy, Name: "atomic.ptr.cast");
10987 }
10988 }
10989 checkAndEmitFlushAfterAtomic(Loc, AO, AK: AtomicKind::Read);
10990 Builder.CreateStore(Val: XRead, Ptr: V.Var, isVolatile: V.IsVolatile);
10991 return Builder.saveIP();
10992}
10993
10994OpenMPIRBuilder::InsertPointTy
10995OpenMPIRBuilder::createAtomicWrite(const LocationDescription &Loc,
10996 AtomicOpValue &X, Value *Expr,
10997 AtomicOrdering AO, InsertPointTy AllocaIP) {
10998 if (!updateToLocation(Loc))
10999 return Loc.IP;
11000
11001 assert(X.Var->getType()->isPointerTy() &&
11002 "OMP Atomic expects a pointer to target memory");
11003 Type *XElemTy = X.ElemTy;
11004 assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||
11005 XElemTy->isPointerTy() || XElemTy->isStructTy()) &&
11006 "OMP atomic write expected a scalar type");
11007
11008 if (XElemTy->isIntegerTy()) {
11009 StoreInst *XSt = Builder.CreateStore(Val: Expr, Ptr: X.Var, isVolatile: X.IsVolatile);
11010 XSt->setAtomic(Ordering: AO);
11011 } else if (XElemTy->isStructTy()) {
11012 LoadInst *OldVal = Builder.CreateLoad(Ty: XElemTy, Ptr: X.Var, Name: "omp.atomic.read");
11013 const DataLayout &DL = OldVal->getModule()->getDataLayout();
11014 unsigned LoadSize = DL.getTypeStoreSize(Ty: XElemTy);
11015 OpenMPIRBuilder::AtomicInfo atomicInfo(
11016 &Builder, XElemTy, LoadSize * 8, LoadSize * 8, OldVal->getAlign(),
11017 OldVal->getAlign(), true /* UseLibcall */, AllocaIP, X.Var);
11018 atomicInfo.EmitAtomicStoreLibcall(AO, Source: Expr);
11019 OldVal->eraseFromParent();
11020 } else {
11021 // We need to bitcast and perform atomic op as integers
11022 IntegerType *IntCastTy =
11023 IntegerType::get(C&: M.getContext(), NumBits: XElemTy->getScalarSizeInBits());
11024 Value *ExprCast =
11025 Builder.CreateBitCast(V: Expr, DestTy: IntCastTy, Name: "atomic.src.int.cast");
11026 StoreInst *XSt = Builder.CreateStore(Val: ExprCast, Ptr: X.Var, isVolatile: X.IsVolatile);
11027 XSt->setAtomic(Ordering: AO);
11028 }
11029
11030 checkAndEmitFlushAfterAtomic(Loc, AO, AK: AtomicKind::Write);
11031 return Builder.saveIP();
11032}
11033
11034OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createAtomicUpdate(
11035 const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X,
11036 Value *Expr, AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp,
11037 AtomicUpdateCallbackTy &UpdateOp, bool IsXBinopExpr,
11038 bool IsIgnoreDenormalMode, bool IsFineGrainedMemory, bool IsRemoteMemory) {
11039 assert(!isConflictIP(Loc.IP, AllocaIP) && "IPs must not be ambiguous");
11040 if (!updateToLocation(Loc))
11041 return Loc.IP;
11042
11043 LLVM_DEBUG({
11044 Type *XTy = X.Var->getType();
11045 assert(XTy->isPointerTy() &&
11046 "OMP Atomic expects a pointer to target memory");
11047 Type *XElemTy = X.ElemTy;
11048 assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||
11049 XElemTy->isPointerTy() || XElemTy->isStructTy()) &&
11050 "OMP atomic update expected a scalar or struct type");
11051 assert((RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) &&
11052 (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) &&
11053 "OpenMP atomic does not support LT or GT operations");
11054 });
11055
11056 Expected<std::pair<Value *, Value *>> AtomicResult = emitAtomicUpdate(
11057 AllocaIP, X: X.Var, XElemTy: X.ElemTy, Expr, AO, RMWOp, UpdateOp, VolatileX: X.IsVolatile,
11058 IsXBinopExpr, IsIgnoreDenormalMode, IsFineGrainedMemory, IsRemoteMemory);
11059 if (!AtomicResult)
11060 return AtomicResult.takeError();
11061 checkAndEmitFlushAfterAtomic(Loc, AO, AK: AtomicKind::Update);
11062 return Builder.saveIP();
11063}
11064
11065// FIXME: Duplicating AtomicExpand
11066Value *OpenMPIRBuilder::emitRMWOpAsInstruction(Value *Src1, Value *Src2,
11067 AtomicRMWInst::BinOp RMWOp) {
11068 switch (RMWOp) {
11069 case AtomicRMWInst::Add:
11070 return Builder.CreateAdd(LHS: Src1, RHS: Src2);
11071 case AtomicRMWInst::Sub:
11072 return Builder.CreateSub(LHS: Src1, RHS: Src2);
11073 case AtomicRMWInst::And:
11074 return Builder.CreateAnd(LHS: Src1, RHS: Src2);
11075 case AtomicRMWInst::Nand:
11076 return Builder.CreateNeg(V: Builder.CreateAnd(LHS: Src1, RHS: Src2));
11077 case AtomicRMWInst::Or:
11078 return Builder.CreateOr(LHS: Src1, RHS: Src2);
11079 case AtomicRMWInst::Xor:
11080 return Builder.CreateXor(LHS: Src1, RHS: Src2);
11081 case AtomicRMWInst::Xchg:
11082 case AtomicRMWInst::FAdd:
11083 case AtomicRMWInst::FSub:
11084 case AtomicRMWInst::BAD_BINOP:
11085 case AtomicRMWInst::Max:
11086 case AtomicRMWInst::Min:
11087 case AtomicRMWInst::UMax:
11088 case AtomicRMWInst::UMin:
11089 case AtomicRMWInst::FMax:
11090 case AtomicRMWInst::FMin:
11091 case AtomicRMWInst::FMaximum:
11092 case AtomicRMWInst::FMinimum:
11093 case AtomicRMWInst::FMaximumNum:
11094 case AtomicRMWInst::FMinimumNum:
11095 case AtomicRMWInst::UIncWrap:
11096 case AtomicRMWInst::UDecWrap:
11097 case AtomicRMWInst::USubCond:
11098 case AtomicRMWInst::USubSat:
11099 llvm_unreachable("Unsupported atomic update operation");
11100 }
11101 llvm_unreachable("Unsupported atomic update operation");
11102}
11103
11104static AtomicOrdering TransformReleaseAcquireRelease(AtomicOrdering AO) {
11105 // Loads cannot use Release or AcquireRelease ordering. This load is
11106 // just the initial value for the cmpxchg loop; the cmpxchg itself
11107 // retains the original ordering.
11108 AtomicOrdering LoadAO = AO;
11109
11110 if (AO == AtomicOrdering::Release) {
11111 LoadAO = AtomicOrdering::Monotonic;
11112 } else if (AO == AtomicOrdering::AcquireRelease) {
11113 LoadAO = AtomicOrdering::Acquire;
11114 }
11115
11116 return LoadAO;
11117}
11118
11119Expected<std::pair<Value *, Value *>> OpenMPIRBuilder::emitAtomicUpdate(
11120 InsertPointTy AllocaIP, Value *X, Type *XElemTy, Value *Expr,
11121 AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp,
11122 AtomicUpdateCallbackTy &UpdateOp, bool VolatileX, bool IsXBinopExpr,
11123 bool IsIgnoreDenormalMode, bool IsFineGrainedMemory, bool IsRemoteMemory) {
11124 // TODO: handle the case where XElemTy is not byte-sized or not a power of 2.
11125 bool emitRMWOp = false;
11126 switch (RMWOp) {
11127 case AtomicRMWInst::Add:
11128 case AtomicRMWInst::And:
11129 case AtomicRMWInst::Nand:
11130 case AtomicRMWInst::Or:
11131 case AtomicRMWInst::Xor:
11132 case AtomicRMWInst::Xchg:
11133 emitRMWOp = XElemTy;
11134 break;
11135 case AtomicRMWInst::Sub:
11136 emitRMWOp = (IsXBinopExpr && XElemTy);
11137 break;
11138 default:
11139 emitRMWOp = false;
11140 }
11141 emitRMWOp &= XElemTy->isIntegerTy();
11142
11143 std::pair<Value *, Value *> Res;
11144 if (emitRMWOp) {
11145 AtomicRMWInst *RMWInst =
11146 Builder.CreateAtomicRMW(Op: RMWOp, Ptr: X, Val: Expr, Align: llvm::MaybeAlign(), Ordering: AO);
11147 if (T.isAMDGPU()) {
11148 if (IsIgnoreDenormalMode)
11149 RMWInst->setMetadata(Kind: "amdgpu.ignore.denormal.mode",
11150 Node: llvm::MDNode::get(Context&: Builder.getContext(), MDs: {}));
11151 if (!IsFineGrainedMemory)
11152 RMWInst->setMetadata(Kind: "amdgpu.no.fine.grained.memory",
11153 Node: llvm::MDNode::get(Context&: Builder.getContext(), MDs: {}));
11154 if (!IsRemoteMemory)
11155 RMWInst->setMetadata(Kind: "amdgpu.no.remote.memory",
11156 Node: llvm::MDNode::get(Context&: Builder.getContext(), MDs: {}));
11157 }
11158 Res.first = RMWInst;
11159 // not needed except in case of postfix captures. Generate anyway for
11160 // consistency with the else part. Will be removed with any DCE pass.
11161 // AtomicRMWInst::Xchg does not have a coressponding instruction.
11162 if (RMWOp == AtomicRMWInst::Xchg)
11163 Res.second = Res.first;
11164 else
11165 Res.second = emitRMWOpAsInstruction(Src1: Res.first, Src2: Expr, RMWOp);
11166 } else if (XElemTy->isStructTy()) {
11167 LoadInst *OldVal =
11168 Builder.CreateLoad(Ty: XElemTy, Ptr: X, Name: X->getName() + ".atomic.load");
11169 AtomicOrdering LoadAO = TransformReleaseAcquireRelease(AO);
11170 OldVal->setAtomic(Ordering: LoadAO);
11171 const DataLayout &LoadDL = OldVal->getModule()->getDataLayout();
11172 unsigned LoadSize = LoadDL.getTypeStoreSize(Ty: XElemTy);
11173
11174 OpenMPIRBuilder::AtomicInfo atomicInfo(
11175 &Builder, XElemTy, LoadSize * 8, LoadSize * 8, OldVal->getAlign(),
11176 OldVal->getAlign(), true /* UseLibcall */, AllocaIP, X);
11177 auto AtomicLoadRes = atomicInfo.EmitAtomicLoadLibcall(AO);
11178 BasicBlock *CurBB = Builder.GetInsertBlock();
11179 Instruction *CurBBTI = CurBB->getTerminatorOrNull();
11180 CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable();
11181 BasicBlock *ExitBB =
11182 CurBB->splitBasicBlock(I: CurBBTI, BBName: X->getName() + ".atomic.exit");
11183 BasicBlock *ContBB = CurBB->splitBasicBlock(I: CurBB->getTerminator(),
11184 BBName: X->getName() + ".atomic.cont");
11185 ContBB->getTerminator()->eraseFromParent();
11186 Builder.restoreIP(IP: AllocaIP);
11187 AllocaInst *NewAtomicAddr = Builder.CreateAlloca(Ty: XElemTy);
11188 NewAtomicAddr->setName(X->getName() + "x.new.val");
11189 Builder.SetInsertPoint(ContBB);
11190 llvm::PHINode *PHI = Builder.CreatePHI(Ty: OldVal->getType(), NumReservedValues: 2);
11191 PHI->addIncoming(V: AtomicLoadRes.first, BB: CurBB);
11192 Value *OldExprVal = PHI;
11193 Expected<Value *> CBResult = UpdateOp(OldExprVal, Builder);
11194 if (!CBResult)
11195 return CBResult.takeError();
11196 Value *Upd = *CBResult;
11197 Builder.CreateStore(Val: Upd, Ptr: NewAtomicAddr);
11198 AtomicOrdering Failure =
11199 llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: AO);
11200 auto Result = atomicInfo.EmitAtomicCompareExchangeLibcall(
11201 ExpectedVal: AtomicLoadRes.second, DesiredVal: NewAtomicAddr, Success: AO, Failure);
11202 LoadInst *PHILoad = Builder.CreateLoad(Ty: XElemTy, Ptr: Result.first);
11203 PHI->addIncoming(V: PHILoad, BB: Builder.GetInsertBlock());
11204 Builder.CreateCondBr(Cond: Result.second, True: ExitBB, False: ContBB);
11205 OldVal->eraseFromParent();
11206 Res.first = OldExprVal;
11207 Res.second = Upd;
11208
11209 if (UnreachableInst *ExitTI =
11210 dyn_cast<UnreachableInst>(Val: ExitBB->getTerminator())) {
11211 CurBBTI->eraseFromParent();
11212 Builder.SetInsertPoint(ExitBB);
11213 } else {
11214 Builder.SetInsertPoint(ExitTI);
11215 }
11216 } else {
11217 IntegerType *IntCastTy =
11218 IntegerType::get(C&: M.getContext(), NumBits: XElemTy->getScalarSizeInBits());
11219 LoadInst *OldVal =
11220 Builder.CreateLoad(Ty: IntCastTy, Ptr: X, Name: X->getName() + ".atomic.load");
11221 AtomicOrdering LoadAO = TransformReleaseAcquireRelease(AO);
11222 OldVal->setAtomic(Ordering: LoadAO);
11223 // CurBB
11224 // | /---\
11225 // ContBB |
11226 // | \---/
11227 // ExitBB
11228 BasicBlock *CurBB = Builder.GetInsertBlock();
11229 Instruction *CurBBTI = CurBB->getTerminatorOrNull();
11230 CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable();
11231 BasicBlock *ExitBB =
11232 CurBB->splitBasicBlock(I: CurBBTI, BBName: X->getName() + ".atomic.exit");
11233 BasicBlock *ContBB = CurBB->splitBasicBlock(I: CurBB->getTerminator(),
11234 BBName: X->getName() + ".atomic.cont");
11235 ContBB->getTerminator()->eraseFromParent();
11236 Builder.restoreIP(IP: AllocaIP);
11237 AllocaInst *NewAtomicAddr = Builder.CreateAlloca(Ty: XElemTy);
11238 NewAtomicAddr->setName(X->getName() + "x.new.val");
11239 Builder.SetInsertPoint(ContBB);
11240 llvm::PHINode *PHI = Builder.CreatePHI(Ty: OldVal->getType(), NumReservedValues: 2);
11241 PHI->addIncoming(V: OldVal, BB: CurBB);
11242 bool IsIntTy = XElemTy->isIntegerTy();
11243 Value *OldExprVal = PHI;
11244 if (!IsIntTy) {
11245 if (XElemTy->isFloatingPointTy()) {
11246 OldExprVal = Builder.CreateBitCast(V: PHI, DestTy: XElemTy,
11247 Name: X->getName() + ".atomic.fltCast");
11248 } else {
11249 OldExprVal = Builder.CreateIntToPtr(V: PHI, DestTy: XElemTy,
11250 Name: X->getName() + ".atomic.ptrCast");
11251 }
11252 }
11253
11254 Expected<Value *> CBResult = UpdateOp(OldExprVal, Builder);
11255 if (!CBResult)
11256 return CBResult.takeError();
11257 Value *Upd = *CBResult;
11258 Builder.CreateStore(Val: Upd, Ptr: NewAtomicAddr);
11259 LoadInst *DesiredVal = Builder.CreateLoad(Ty: IntCastTy, Ptr: NewAtomicAddr);
11260 AtomicOrdering Failure =
11261 llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: AO);
11262 AtomicCmpXchgInst *Result = Builder.CreateAtomicCmpXchg(
11263 Ptr: X, Cmp: PHI, New: DesiredVal, Align: llvm::MaybeAlign(), SuccessOrdering: AO, FailureOrdering: Failure);
11264 Result->setVolatile(VolatileX);
11265 Value *PreviousVal = Builder.CreateExtractValue(Agg: Result, /*Idxs=*/0);
11266 Value *SuccessFailureVal = Builder.CreateExtractValue(Agg: Result, /*Idxs=*/1);
11267 PHI->addIncoming(V: PreviousVal, BB: Builder.GetInsertBlock());
11268 Builder.CreateCondBr(Cond: SuccessFailureVal, True: ExitBB, False: ContBB);
11269
11270 Res.first = OldExprVal;
11271 Res.second = Upd;
11272
11273 // set Insertion point in exit block
11274 if (UnreachableInst *ExitTI =
11275 dyn_cast<UnreachableInst>(Val: ExitBB->getTerminator())) {
11276 CurBBTI->eraseFromParent();
11277 Builder.SetInsertPoint(ExitBB);
11278 } else {
11279 Builder.SetInsertPoint(ExitTI);
11280 }
11281 }
11282
11283 return Res;
11284}
11285
11286OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createAtomicCapture(
11287 const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X,
11288 AtomicOpValue &V, Value *Expr, AtomicOrdering AO,
11289 AtomicRMWInst::BinOp RMWOp, AtomicUpdateCallbackTy &UpdateOp,
11290 bool UpdateExpr, bool IsPostfixUpdate, bool IsXBinopExpr,
11291 bool IsIgnoreDenormalMode, bool IsFineGrainedMemory, bool IsRemoteMemory) {
11292 if (!updateToLocation(Loc))
11293 return Loc.IP;
11294
11295 LLVM_DEBUG({
11296 Type *XTy = X.Var->getType();
11297 assert(XTy->isPointerTy() &&
11298 "OMP Atomic expects a pointer to target memory");
11299 Type *XElemTy = X.ElemTy;
11300 assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||
11301 XElemTy->isPointerTy() || XElemTy->isStructTy()) &&
11302 "OMP atomic capture expected a scalar or struct type");
11303 assert((RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) &&
11304 "OpenMP atomic does not support LT or GT operations");
11305 });
11306
11307 // If UpdateExpr is 'x' updated with some `expr` not based on 'x',
11308 // 'x' is simply atomically rewritten with 'expr'.
11309 AtomicRMWInst::BinOp AtomicOp = (UpdateExpr ? RMWOp : AtomicRMWInst::Xchg);
11310 Expected<std::pair<Value *, Value *>> AtomicResult = emitAtomicUpdate(
11311 AllocaIP, X: X.Var, XElemTy: X.ElemTy, Expr, AO, RMWOp: AtomicOp, UpdateOp, VolatileX: X.IsVolatile,
11312 IsXBinopExpr, IsIgnoreDenormalMode, IsFineGrainedMemory, IsRemoteMemory);
11313 if (!AtomicResult)
11314 return AtomicResult.takeError();
11315 Value *CapturedVal =
11316 (IsPostfixUpdate ? AtomicResult->first : AtomicResult->second);
11317 Builder.CreateStore(Val: CapturedVal, Ptr: V.Var, isVolatile: V.IsVolatile);
11318
11319 checkAndEmitFlushAfterAtomic(Loc, AO, AK: AtomicKind::Capture);
11320 return Builder.saveIP();
11321}
11322
11323OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicCompare(
11324 const LocationDescription &Loc, AtomicOpValue &X, AtomicOpValue &V,
11325 AtomicOpValue &R, Value *E, Value *D, AtomicOrdering AO,
11326 omp::OMPAtomicCompareOp Op, bool IsXBinopExpr, bool IsPostfixUpdate,
11327 bool IsFailOnly, bool IsWeak) {
11328
11329 AtomicOrdering Failure = AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: AO);
11330 return createAtomicCompare(Loc, X, V, R, E, D, AO, Op, IsXBinopExpr,
11331 IsPostfixUpdate, IsFailOnly, Failure, IsWeak);
11332}
11333
11334OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicCompare(
11335 const LocationDescription &Loc, AtomicOpValue &X, AtomicOpValue &V,
11336 AtomicOpValue &R, Value *E, Value *D, AtomicOrdering AO,
11337 omp::OMPAtomicCompareOp Op, bool IsXBinopExpr, bool IsPostfixUpdate,
11338 bool IsFailOnly, AtomicOrdering Failure, bool IsWeak) {
11339
11340 if (!updateToLocation(Loc))
11341 return Loc.IP;
11342
11343 assert(X.Var->getType()->isPointerTy() &&
11344 "OMP atomic expects a pointer to target memory");
11345 // compare capture
11346 if (V.Var) {
11347 assert(V.Var->getType()->isPointerTy() && "v.var must be of pointer type");
11348 assert(V.ElemTy == X.ElemTy && "x and v must be of same type");
11349 }
11350
11351 bool IsInteger = E->getType()->isIntegerTy();
11352
11353 if (Op == OMPAtomicCompareOp::EQ) {
11354 // OldValue and SuccessOrFail are set below and used in the shared V.Var /
11355 // R.Var handling.
11356 Value *OldValue = nullptr;
11357 Value *SuccessOrFail = nullptr;
11358
11359 if (!IsInteger && HandleFPNegZero) {
11360 // IEEE 754 special cases for cmpxchg (which is bitwise):
11361 // 1. -0.0 == +0.0 but they have different bit patterns.
11362 // 2. NaN != NaN but identical NaN bit patterns would match.
11363 //
11364 // CurBB:
11365 // %e_int = bitcast E to intN
11366 // %d_int = bitcast D to intN
11367 // %x_curr = load atomic intN, X
11368 // %x_fp = bitcast %x_curr to FP
11369 // %e_is_nan = fcmp uno E, E
11370 // %x_is_nan = fcmp uno %x_fp, %x_fp
11371 // %either_nan = or %e_is_nan, %x_is_nan
11372 // br %either_nan, NaNBB, NotNaNBB
11373 // NaNBB: ; NaN == anything is always false
11374 // br ExitBB
11375 // NotNaNBB:
11376 // %x_is_zero = fcmp oeq %x_fp, 0.0
11377 // %e_is_zero = fcmp oeq E, 0.0
11378 // %both_zero = and %x_is_zero, %e_is_zero
11379 // br %both_zero, ZeroBB, NormalBB
11380 // ZeroBB: ; both ±0.0 → x = d
11381 // cmpxchg X, %x_curr, %d_int
11382 // br ExitBB
11383 // NormalBB: ; original path
11384 // cmpxchg X, %e_int, %d_int
11385 // br ExitBB
11386 // ExitBB:
11387 // phi merge
11388 IntegerType *IntCastTy =
11389 IntegerType::get(C&: M.getContext(), NumBits: X.ElemTy->getScalarSizeInBits());
11390 Value *EBCast = Builder.CreateBitCast(V: E, DestTy: IntCastTy);
11391 Value *DBCast = Builder.CreateBitCast(V: D, DestTy: IntCastTy);
11392
11393 // Load X atomically.
11394 LoadInst *XCurr = Builder.CreateLoad(Ty: IntCastTy, Ptr: X.Var,
11395 Name: X.Var->getName() + ".atomic.load");
11396 XCurr->setAtomic(Ordering: AtomicOrdering::Monotonic);
11397 Value *XFP = Builder.CreateBitCast(V: XCurr, DestTy: X.ElemTy);
11398
11399 // IEEE 754: NaN != NaN, but cmpxchg would succeed if E and X have
11400 // the same NaN bit pattern. Skip cmpxchg when either is NaN.
11401 Value *EIsNaN = Builder.CreateFCmpUNO(LHS: E, RHS: E, Name: "atomic.e.isnan");
11402 Value *XIsNaN = Builder.CreateFCmpUNO(LHS: XFP, RHS: XFP, Name: "atomic.x.isnan");
11403 Value *EitherNaN = Builder.CreateOr(LHS: EIsNaN, RHS: XIsNaN, Name: "atomic.either.nan");
11404
11405 BasicBlock *CurBB = Builder.GetInsertBlock();
11406 Function *F = CurBB->getParent();
11407 Instruction *CurBBTI = CurBB->getTerminatorOrNull();
11408 CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable();
11409 BasicBlock *ExitBB =
11410 CurBB->splitBasicBlock(I: CurBBTI, BBName: X.Var->getName() + ".atomic.exit");
11411 BasicBlock *NaNBB = BasicBlock::Create(
11412 Context&: M.getContext(), Name: X.Var->getName() + ".atomic.nan", Parent: F, InsertBefore: ExitBB);
11413 BasicBlock *NotNaNBB = BasicBlock::Create(
11414 Context&: M.getContext(), Name: X.Var->getName() + ".atomic.notnan", Parent: F, InsertBefore: ExitBB);
11415 BasicBlock *ZeroBB = BasicBlock::Create(
11416 Context&: M.getContext(), Name: X.Var->getName() + ".atomic.zero", Parent: F, InsertBefore: ExitBB);
11417 BasicBlock *NormalBB = BasicBlock::Create(
11418 Context&: M.getContext(), Name: X.Var->getName() + ".atomic.normal", Parent: F, InsertBefore: ExitBB);
11419
11420 // If either E or X is NaN → NaNBB (always fails), else check for ±0.0.
11421 CurBB->getTerminator()->eraseFromParent();
11422 Builder.SetInsertPoint(CurBB);
11423 Builder.CreateCondBr(Cond: EitherNaN, True: NaNBB, False: NotNaNBB);
11424
11425 // NaNBB: NaN == anything is always false; skip cmpxchg.
11426 Builder.SetInsertPoint(NaNBB);
11427 Builder.CreateBr(Dest: ExitBB);
11428
11429 // NotNaNBB: check both X and E for ±0.0.
11430 Builder.SetInsertPoint(NotNaNBB);
11431 Value *XIsZero =
11432 Builder.CreateFCmpOEQ(LHS: XFP, RHS: ConstantFP::getZero(Ty: X.ElemTy),
11433 Name: X.Var->getName() + ".atomic.xiszero");
11434 Value *EIsZero = Builder.CreateFCmpOEQ(LHS: E, RHS: ConstantFP::getZero(Ty: X.ElemTy),
11435 Name: "atomic.e.iszero");
11436 Value *BothZero = Builder.CreateAnd(LHS: XIsZero, RHS: EIsZero, Name: "atomic.both.zero");
11437 Builder.CreateCondBr(Cond: BothZero, True: ZeroBB, False: NormalBB);
11438
11439 // ZeroBB: cmpxchg with X's loaded bit-pattern.
11440 Builder.SetInsertPoint(ZeroBB);
11441 AtomicCmpXchgInst *ResZero = Builder.CreateAtomicCmpXchg(
11442 Ptr: X.Var, Cmp: XCurr, New: DBCast, Align: MaybeAlign(), SuccessOrdering: AO, FailureOrdering: Failure);
11443 ResZero->setWeak(IsWeak);
11444 Value *OldZero = Builder.CreateExtractValue(Agg: ResZero, /*Idxs=*/0);
11445 Value *OkZero = Builder.CreateExtractValue(Agg: ResZero, /*Idxs=*/1);
11446 Builder.CreateBr(Dest: ExitBB);
11447
11448 // NormalBB: original bitwise cmpxchg.
11449 Builder.SetInsertPoint(NormalBB);
11450 AtomicCmpXchgInst *ResNormal = Builder.CreateAtomicCmpXchg(
11451 Ptr: X.Var, Cmp: EBCast, New: DBCast, Align: MaybeAlign(), SuccessOrdering: AO, FailureOrdering: Failure);
11452 ResNormal->setWeak(IsWeak);
11453 Value *OldNormal = Builder.CreateExtractValue(Agg: ResNormal, /*Idxs=*/0);
11454 Value *OkNormal = Builder.CreateExtractValue(Agg: ResNormal, /*Idxs=*/1);
11455 Builder.CreateBr(Dest: ExitBB);
11456
11457 // ExitBB: merge results from NaN, Zero, and Normal paths.
11458 Builder.SetInsertPoint(TheBB: ExitBB, IP: ExitBB->begin());
11459 PHINode *OldIntPHI =
11460 Builder.CreatePHI(Ty: IntCastTy, NumReservedValues: 3, Name: X.Var->getName() + ".atomic.old");
11461 OldIntPHI->addIncoming(V: XCurr, BB: NaNBB);
11462 OldIntPHI->addIncoming(V: OldZero, BB: ZeroBB);
11463 OldIntPHI->addIncoming(V: OldNormal, BB: NormalBB);
11464 PHINode *SuccessPHI = Builder.CreatePHI(Ty: Builder.getInt1Ty(), NumReservedValues: 3,
11465 Name: X.Var->getName() + ".atomic.ok");
11466 SuccessPHI->addIncoming(V: Builder.getFalse(), BB: NaNBB);
11467 SuccessPHI->addIncoming(V: OkZero, BB: ZeroBB);
11468 SuccessPHI->addIncoming(V: OkNormal, BB: NormalBB);
11469
11470 if (isa<UnreachableInst>(Val: ExitBB->getTerminator())) {
11471 CurBBTI->eraseFromParent();
11472 Builder.SetInsertPoint(ExitBB);
11473 } else {
11474 Builder.SetInsertPoint(&*ExitBB->getFirstNonPHIIt());
11475 }
11476
11477 OldValue = Builder.CreateBitCast(V: OldIntPHI, DestTy: X.ElemTy,
11478 Name: X.Var->getName() + ".atomic.old.fp");
11479 SuccessOrFail = SuccessPHI;
11480 } else {
11481 AtomicCmpXchgInst *Result = nullptr;
11482 if (!IsInteger) {
11483 IntegerType *IntCastTy =
11484 IntegerType::get(C&: M.getContext(), NumBits: X.ElemTy->getScalarSizeInBits());
11485 Value *EBCast = Builder.CreateBitCast(V: E, DestTy: IntCastTy);
11486 Value *DBCast = Builder.CreateBitCast(V: D, DestTy: IntCastTy);
11487 Result = Builder.CreateAtomicCmpXchg(Ptr: X.Var, Cmp: EBCast, New: DBCast,
11488 Align: MaybeAlign(), SuccessOrdering: AO, FailureOrdering: Failure);
11489 } else {
11490 Result =
11491 Builder.CreateAtomicCmpXchg(Ptr: X.Var, Cmp: E, New: D, Align: MaybeAlign(), SuccessOrdering: AO, FailureOrdering: Failure);
11492 }
11493 Result->setWeak(IsWeak);
11494
11495 if (V.Var) {
11496 OldValue = Builder.CreateExtractValue(Agg: Result, /*Idxs=*/0);
11497 if (!IsInteger)
11498 OldValue = Builder.CreateBitCast(V: OldValue, DestTy: X.ElemTy);
11499 assert(OldValue->getType() == V.ElemTy &&
11500 "OldValue and V must be of same type");
11501 if (IsPostfixUpdate) {
11502 Builder.CreateStore(Val: OldValue, Ptr: V.Var, isVolatile: V.IsVolatile);
11503 } else {
11504 SuccessOrFail = Builder.CreateExtractValue(Agg: Result, /*Idxs=*/1);
11505 if (IsFailOnly) {
11506 BasicBlock *CurBB = Builder.GetInsertBlock();
11507 Instruction *CurBBTI = CurBB->getTerminatorOrNull();
11508 CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable();
11509 BasicBlock *ExitBB = CurBB->splitBasicBlock(
11510 I: CurBBTI, BBName: X.Var->getName() + ".atomic.exit");
11511 BasicBlock *ContBB = CurBB->splitBasicBlock(
11512 I: CurBB->getTerminator(), BBName: X.Var->getName() + ".atomic.cont");
11513 ContBB->getTerminator()->eraseFromParent();
11514 CurBB->getTerminator()->eraseFromParent();
11515
11516 Builder.CreateCondBr(Cond: SuccessOrFail, True: ExitBB, False: ContBB);
11517
11518 Builder.SetInsertPoint(ContBB);
11519 Builder.CreateStore(Val: OldValue, Ptr: V.Var);
11520 Builder.CreateBr(Dest: ExitBB);
11521
11522 if (UnreachableInst *ExitTI =
11523 dyn_cast<UnreachableInst>(Val: ExitBB->getTerminator())) {
11524 CurBBTI->eraseFromParent();
11525 Builder.SetInsertPoint(ExitBB);
11526 } else {
11527 Builder.SetInsertPoint(ExitTI);
11528 }
11529 } else {
11530 Value *CapturedValue =
11531 Builder.CreateSelect(C: SuccessOrFail, True: E, False: OldValue);
11532 Builder.CreateStore(Val: CapturedValue, Ptr: V.Var, isVolatile: V.IsVolatile);
11533 }
11534 }
11535 }
11536 // The comparison result has to be stored.
11537 if (R.Var) {
11538 assert(R.Var->getType()->isPointerTy() &&
11539 "r.var must be of pointer type");
11540 assert(R.ElemTy->isIntegerTy() && "r must be of integral type");
11541
11542 Value *SuccessFailureVal =
11543 Builder.CreateExtractValue(Agg: Result, /*Idxs=*/1);
11544 Value *ResultCast =
11545 R.IsSigned ? Builder.CreateSExt(V: SuccessFailureVal, DestTy: R.ElemTy)
11546 : Builder.CreateZExt(V: SuccessFailureVal, DestTy: R.ElemTy);
11547 Builder.CreateStore(Val: ResultCast, Ptr: R.Var, isVolatile: R.IsVolatile);
11548 }
11549 }
11550
11551 // For the HandleFPNegZero path, handle V.Var and R.Var using the
11552 // pre-computed OldValue and SuccessOrFail.
11553 if (HandleFPNegZero && !IsInteger) {
11554 if (V.Var) {
11555 assert(OldValue->getType() == V.ElemTy &&
11556 "OldValue and V must be of same type");
11557 if (IsPostfixUpdate) {
11558 Builder.CreateStore(Val: OldValue, Ptr: V.Var, isVolatile: V.IsVolatile);
11559 } else {
11560 if (IsFailOnly) {
11561 BasicBlock *CurBB = Builder.GetInsertBlock();
11562 Instruction *CurBBTI = CurBB->getTerminatorOrNull();
11563 CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable();
11564 BasicBlock *ExitBB = CurBB->splitBasicBlock(
11565 I: CurBBTI, BBName: X.Var->getName() + ".atomic.exit");
11566 BasicBlock *ContBB = CurBB->splitBasicBlock(
11567 I: CurBB->getTerminator(), BBName: X.Var->getName() + ".atomic.cont");
11568 ContBB->getTerminator()->eraseFromParent();
11569 CurBB->getTerminator()->eraseFromParent();
11570
11571 Builder.CreateCondBr(Cond: SuccessOrFail, True: ExitBB, False: ContBB);
11572
11573 Builder.SetInsertPoint(ContBB);
11574 Builder.CreateStore(Val: OldValue, Ptr: V.Var);
11575 Builder.CreateBr(Dest: ExitBB);
11576
11577 if (UnreachableInst *ExitTI =
11578 dyn_cast<UnreachableInst>(Val: ExitBB->getTerminator())) {
11579 CurBBTI->eraseFromParent();
11580 Builder.SetInsertPoint(ExitBB);
11581 } else {
11582 Builder.SetInsertPoint(ExitTI);
11583 }
11584 } else {
11585 Value *CapturedValue =
11586 Builder.CreateSelect(C: SuccessOrFail, True: E, False: OldValue);
11587 Builder.CreateStore(Val: CapturedValue, Ptr: V.Var, isVolatile: V.IsVolatile);
11588 }
11589 }
11590 }
11591 // The comparison result has to be stored.
11592 if (R.Var) {
11593 assert(R.Var->getType()->isPointerTy() &&
11594 "r.var must be of pointer type");
11595 assert(R.ElemTy->isIntegerTy() && "r must be of integral type");
11596
11597 Value *ResultCast = R.IsSigned
11598 ? Builder.CreateSExt(V: SuccessOrFail, DestTy: R.ElemTy)
11599 : Builder.CreateZExt(V: SuccessOrFail, DestTy: R.ElemTy);
11600 Builder.CreateStore(Val: ResultCast, Ptr: R.Var, isVolatile: R.IsVolatile);
11601 }
11602 }
11603 } else {
11604 assert((Op == OMPAtomicCompareOp::MAX || Op == OMPAtomicCompareOp::MIN) &&
11605 "Op should be either max or min at this point");
11606 assert(!IsFailOnly && "IsFailOnly is only valid when the comparison is ==");
11607
11608 // Reverse the ordop as the OpenMP forms are different from LLVM forms.
11609 // Let's take max as example.
11610 // OpenMP form:
11611 // x = x > expr ? expr : x;
11612 // LLVM form:
11613 // *ptr = *ptr > val ? *ptr : val;
11614 // We need to transform to LLVM form.
11615 // x = x <= expr ? x : expr;
11616 AtomicRMWInst::BinOp NewOp;
11617 if (IsXBinopExpr) {
11618 if (IsInteger) {
11619 if (X.IsSigned)
11620 NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::Min
11621 : AtomicRMWInst::Max;
11622 else
11623 NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::UMin
11624 : AtomicRMWInst::UMax;
11625 } else {
11626 NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::FMin
11627 : AtomicRMWInst::FMax;
11628 }
11629 } else {
11630 if (IsInteger) {
11631 if (X.IsSigned)
11632 NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::Max
11633 : AtomicRMWInst::Min;
11634 else
11635 NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::UMax
11636 : AtomicRMWInst::UMin;
11637 } else {
11638 NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::FMax
11639 : AtomicRMWInst::FMin;
11640 }
11641 }
11642
11643 AtomicRMWInst *OldValue =
11644 Builder.CreateAtomicRMW(Op: NewOp, Ptr: X.Var, Val: E, Align: MaybeAlign(), Ordering: AO);
11645 if (V.Var) {
11646 Value *CapturedValue = nullptr;
11647 if (IsPostfixUpdate) {
11648 CapturedValue = OldValue;
11649 } else {
11650 CmpInst::Predicate Pred;
11651 switch (NewOp) {
11652 case AtomicRMWInst::Max:
11653 Pred = CmpInst::ICMP_SGT;
11654 break;
11655 case AtomicRMWInst::UMax:
11656 Pred = CmpInst::ICMP_UGT;
11657 break;
11658 case AtomicRMWInst::FMax:
11659 Pred = CmpInst::FCMP_OGT;
11660 break;
11661 case AtomicRMWInst::Min:
11662 Pred = CmpInst::ICMP_SLT;
11663 break;
11664 case AtomicRMWInst::UMin:
11665 Pred = CmpInst::ICMP_ULT;
11666 break;
11667 case AtomicRMWInst::FMin:
11668 Pred = CmpInst::FCMP_OLT;
11669 break;
11670 default:
11671 llvm_unreachable("unexpected comparison op");
11672 }
11673 Value *NonAtomicCmp = Builder.CreateCmp(Pred, LHS: OldValue, RHS: E);
11674 CapturedValue = Builder.CreateSelect(C: NonAtomicCmp, True: E, False: OldValue);
11675 }
11676 Builder.CreateStore(Val: CapturedValue, Ptr: V.Var, isVolatile: V.IsVolatile);
11677 }
11678 }
11679
11680 checkAndEmitFlushAfterAtomic(Loc, AO, AK: AtomicKind::Compare);
11681
11682 return Builder.saveIP();
11683}
11684
11685OpenMPIRBuilder::InsertPointOrErrorTy
11686OpenMPIRBuilder::createTeams(const LocationDescription &Loc,
11687 BodyGenCallbackTy BodyGenCB, Value *NumTeamsLower,
11688 Value *NumTeamsUpper, Value *ThreadLimit,
11689 Value *IfExpr) {
11690 if (!updateToLocation(Loc))
11691 return InsertPointTy();
11692
11693 uint32_t SrcLocStrSize;
11694 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
11695 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
11696 Function *CurrentFunction = Builder.GetInsertBlock()->getParent();
11697
11698 // Outer allocation basicblock is the entry block of the current function.
11699 BasicBlock &OuterAllocaBB = CurrentFunction->getEntryBlock();
11700 if (&OuterAllocaBB == Builder.GetInsertBlock()) {
11701 BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, Name: "teams.entry");
11702 Builder.SetInsertPoint(TheBB: BodyBB, IP: BodyBB->begin());
11703 }
11704
11705 // The current basic block is split into four basic blocks. After outlining,
11706 // they will be mapped as follows:
11707 // ```
11708 // def current_fn() {
11709 // current_basic_block:
11710 // br label %teams.exit
11711 // teams.exit:
11712 // ; instructions after teams
11713 // }
11714 //
11715 // def outlined_fn() {
11716 // teams.alloca:
11717 // br label %teams.body
11718 // teams.body:
11719 // ; instructions within teams body
11720 // }
11721 // ```
11722 BasicBlock *ExitBB = splitBB(Builder, /*CreateBranch=*/true, Name: "teams.exit");
11723 BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, Name: "teams.body");
11724 BasicBlock *AllocaBB =
11725 splitBB(Builder, /*CreateBranch=*/true, Name: "teams.alloca");
11726
11727 bool SubClausesPresent =
11728 (NumTeamsLower || NumTeamsUpper || ThreadLimit || IfExpr);
11729 // Push num_teams
11730 if (!Config.isTargetDevice() && SubClausesPresent) {
11731 assert((NumTeamsLower == nullptr || NumTeamsUpper != nullptr) &&
11732 "if lowerbound is non-null, then upperbound must also be non-null "
11733 "for bounds on num_teams");
11734
11735 if (NumTeamsUpper == nullptr)
11736 NumTeamsUpper = Builder.getInt32(C: 0);
11737
11738 if (NumTeamsLower == nullptr)
11739 NumTeamsLower = NumTeamsUpper;
11740
11741 if (IfExpr) {
11742 assert(IfExpr->getType()->isIntegerTy() &&
11743 "argument to if clause must be an integer value");
11744
11745 // upper = ifexpr ? upper : 1
11746 if (IfExpr->getType() != Int1)
11747 IfExpr = Builder.CreateICmpNE(LHS: IfExpr,
11748 RHS: ConstantInt::get(Ty: IfExpr->getType(), V: 0));
11749 NumTeamsUpper = Builder.CreateSelect(
11750 C: IfExpr, True: NumTeamsUpper, False: Builder.getInt32(C: 1), Name: "numTeamsUpper");
11751
11752 // lower = ifexpr ? lower : 1
11753 NumTeamsLower = Builder.CreateSelect(
11754 C: IfExpr, True: NumTeamsLower, False: Builder.getInt32(C: 1), Name: "numTeamsLower");
11755 }
11756
11757 if (ThreadLimit == nullptr)
11758 ThreadLimit = Builder.getInt32(C: 0);
11759
11760 // The __kmpc_push_num_teams_51 function expects int32 as the arguments. So,
11761 // truncate or sign extend the passed values to match the int32 parameters.
11762 Value *NumTeamsLowerInt32 =
11763 Builder.CreateSExtOrTrunc(V: NumTeamsLower, DestTy: Builder.getInt32Ty());
11764 Value *NumTeamsUpperInt32 =
11765 Builder.CreateSExtOrTrunc(V: NumTeamsUpper, DestTy: Builder.getInt32Ty());
11766 Value *ThreadLimitInt32 =
11767 Builder.CreateSExtOrTrunc(V: ThreadLimit, DestTy: Builder.getInt32Ty());
11768
11769 Value *ThreadNum = getOrCreateThreadID(Ident);
11770
11771 createRuntimeFunctionCall(
11772 Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_push_num_teams_51),
11773 Args: {Ident, ThreadNum, NumTeamsLowerInt32, NumTeamsUpperInt32,
11774 ThreadLimitInt32});
11775 }
11776 // Generate the body of teams.
11777 InsertPointTy AllocaIP(AllocaBB, AllocaBB->begin());
11778 InsertPointTy CodeGenIP(BodyBB, BodyBB->begin());
11779 if (Error Err = BodyGenCB(AllocaIP, CodeGenIP, ExitBB))
11780 return Err;
11781
11782 auto OI = std::make_unique<OutlineInfo>();
11783 OI->EntryBB = AllocaBB;
11784 OI->ExitBB = ExitBB;
11785 OI->OuterAllocBB = &OuterAllocaBB;
11786
11787 // Insert fake values for global tid and bound tid.
11788 SmallVector<Instruction *, 8> ToBeDeleted;
11789 InsertPointTy OuterAllocaIP(&OuterAllocaBB, OuterAllocaBB.begin());
11790 OI->ExcludeArgsFromAggregate.push_back(Elt: createFakeIntVal(
11791 Builder, OuterAllocaIP, ToBeDeleted, InnerAllocaIP: AllocaIP, Name: "gid", AsPtr: true));
11792 OI->ExcludeArgsFromAggregate.push_back(Elt: createFakeIntVal(
11793 Builder, OuterAllocaIP, ToBeDeleted, InnerAllocaIP: AllocaIP, Name: "tid", AsPtr: true));
11794
11795 auto HostPostOutlineCB = [this, Ident,
11796 ToBeDeleted](Function &OutlinedFn) mutable {
11797 // The stale call instruction will be replaced with a new call instruction
11798 // for runtime call with the outlined function.
11799
11800 assert(OutlinedFn.hasOneUse() &&
11801 "there must be a single user for the outlined function");
11802 CallInst *StaleCI = cast<CallInst>(Val: OutlinedFn.user_back());
11803 ToBeDeleted.push_back(Elt: StaleCI);
11804
11805 assert((OutlinedFn.arg_size() == 2 || OutlinedFn.arg_size() == 3) &&
11806 "Outlined function must have two or three arguments only");
11807
11808 bool HasShared = OutlinedFn.arg_size() == 3;
11809
11810 OutlinedFn.getArg(i: 0)->setName("global.tid.ptr");
11811 OutlinedFn.getArg(i: 1)->setName("bound.tid.ptr");
11812 if (HasShared)
11813 OutlinedFn.getArg(i: 2)->setName("data");
11814
11815 // Call to the runtime function for teams in the current function.
11816 assert(StaleCI && "Error while outlining - no CallInst user found for the "
11817 "outlined function.");
11818 Builder.SetInsertPoint(StaleCI);
11819 SmallVector<Value *> Args = {
11820 Ident, Builder.getInt32(C: StaleCI->arg_size() - 2), &OutlinedFn};
11821 if (HasShared)
11822 Args.push_back(Elt: StaleCI->getArgOperand(i: 2));
11823 createRuntimeFunctionCall(
11824 Callee: getOrCreateRuntimeFunctionPtr(
11825 FnID: omp::RuntimeFunction::OMPRTL___kmpc_fork_teams),
11826 Args);
11827
11828 for (Instruction *I : llvm::reverse(C&: ToBeDeleted))
11829 I->eraseFromParent();
11830 };
11831
11832 if (!Config.isTargetDevice())
11833 OI->PostOutlineCB = HostPostOutlineCB;
11834
11835 addOutlineInfo(OI: std::move(OI));
11836
11837 Builder.SetInsertPoint(ExitBB);
11838
11839 return Builder.saveIP();
11840}
11841
11842OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createDistribute(
11843 const LocationDescription &Loc, InsertPointTy OuterAllocIP,
11844 ArrayRef<BasicBlock *> OuterDeallocBlocks, BodyGenCallbackTy BodyGenCB) {
11845 if (!updateToLocation(Loc))
11846 return InsertPointTy();
11847
11848 BasicBlock *OuterAllocaBB = OuterAllocIP.getBlock();
11849
11850 if (OuterAllocaBB == Builder.GetInsertBlock()) {
11851 BasicBlock *BodyBB =
11852 splitBB(Builder, /*CreateBranch=*/true, Name: "distribute.entry");
11853 Builder.SetInsertPoint(TheBB: BodyBB, IP: BodyBB->begin());
11854 }
11855 BasicBlock *ExitBB =
11856 splitBB(Builder, /*CreateBranch=*/true, Name: "distribute.exit");
11857 BasicBlock *BodyBB =
11858 splitBB(Builder, /*CreateBranch=*/true, Name: "distribute.body");
11859 BasicBlock *AllocaBB =
11860 splitBB(Builder, /*CreateBranch=*/true, Name: "distribute.alloca");
11861
11862 // Generate the body of distribute clause
11863 InsertPointTy AllocaIP(AllocaBB, AllocaBB->begin());
11864 InsertPointTy CodeGenIP(BodyBB, BodyBB->begin());
11865 if (Error Err = BodyGenCB(AllocaIP, CodeGenIP, ExitBB))
11866 return Err;
11867
11868 // When using target we use different runtime functions which require a
11869 // callback.
11870 if (Config.isTargetDevice()) {
11871 auto OI = std::make_unique<OutlineInfo>();
11872 OI->OuterAllocBB = OuterAllocIP.getBlock();
11873 OI->EntryBB = AllocaBB;
11874 OI->ExitBB = ExitBB;
11875 OI->OuterDeallocBBs.reserve(N: OuterDeallocBlocks.size());
11876 copy(Range&: OuterDeallocBlocks, Out: OI->OuterDeallocBBs.end());
11877
11878 addOutlineInfo(OI: std::move(OI));
11879 }
11880 Builder.SetInsertPoint(ExitBB);
11881
11882 return Builder.saveIP();
11883}
11884
11885GlobalVariable *
11886OpenMPIRBuilder::createOffloadMapnames(SmallVectorImpl<llvm::Constant *> &Names,
11887 std::string VarName) {
11888 llvm::Constant *MapNamesArrayInit = llvm::ConstantArray::get(
11889 T: llvm::ArrayType::get(ElementType: llvm::PointerType::getUnqual(C&: M.getContext()),
11890 NumElements: Names.size()),
11891 V: Names);
11892 auto *MapNamesArrayGlobal = new llvm::GlobalVariable(
11893 M, MapNamesArrayInit->getType(),
11894 /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage, MapNamesArrayInit,
11895 VarName);
11896 return MapNamesArrayGlobal;
11897}
11898
11899// Create all simple and struct types exposed by the runtime and remember
11900// the llvm::PointerTypes of them for easy access later.
11901void OpenMPIRBuilder::initializeTypes(Module &M) {
11902 LLVMContext &Ctx = M.getContext();
11903 StructType *T;
11904 unsigned DefaultTargetAS = Config.getDefaultTargetAS();
11905 unsigned ProgramAS = M.getDataLayout().getProgramAddressSpace();
11906#define OMP_TYPE(VarName, InitValue) VarName = InitValue;
11907#define OMP_ARRAY_TYPE(VarName, ElemTy, ArraySize) \
11908 VarName##Ty = ArrayType::get(ElemTy, ArraySize); \
11909 VarName##PtrTy = PointerType::get(Ctx, DefaultTargetAS);
11910#define OMP_FUNCTION_TYPE(VarName, IsVarArg, ReturnType, ...) \
11911 VarName = FunctionType::get(ReturnType, {__VA_ARGS__}, IsVarArg); \
11912 VarName##Ptr = PointerType::get(Ctx, ProgramAS);
11913#define OMP_STRUCT_TYPE(VarName, StructName, Packed, ...) \
11914 T = StructType::getTypeByName(Ctx, StructName); \
11915 if (!T) \
11916 T = StructType::create(Ctx, {__VA_ARGS__}, StructName, Packed); \
11917 VarName = T; \
11918 VarName##Ptr = PointerType::get(Ctx, DefaultTargetAS);
11919#include "llvm/Frontend/OpenMP/OMPKinds.def"
11920}
11921
11922void OpenMPIRBuilder::OutlineInfo::collectBlocks(
11923 SmallPtrSetImpl<BasicBlock *> &BlockSet,
11924 SmallVectorImpl<BasicBlock *> &BlockVector) {
11925 SmallVector<BasicBlock *, 32> Worklist;
11926 BlockSet.insert(Ptr: EntryBB);
11927 BlockSet.insert(Ptr: ExitBB);
11928
11929 Worklist.push_back(Elt: EntryBB);
11930 while (!Worklist.empty()) {
11931 BasicBlock *BB = Worklist.pop_back_val();
11932 BlockVector.push_back(Elt: BB);
11933 for (BasicBlock *SuccBB : successors(BB))
11934 if (BlockSet.insert(Ptr: SuccBB).second)
11935 Worklist.push_back(Elt: SuccBB);
11936 }
11937}
11938
11939std::unique_ptr<CodeExtractor>
11940OpenMPIRBuilder::OutlineInfo::createCodeExtractor(ArrayRef<BasicBlock *> Blocks,
11941 bool ArgsInZeroAddressSpace,
11942 Twine Suffix) {
11943 return std::make_unique<CodeExtractor>(
11944 args&: Blocks, /* DominatorTree */ args: nullptr,
11945 /* AggregateArgs */ args: true,
11946 /* BlockFrequencyInfo */ args: nullptr,
11947 /* BranchProbabilityInfo */ args: nullptr,
11948 /* AssumptionCache */ args: nullptr,
11949 /* AllowVarArgs */ args: true,
11950 /* AllowAlloca */ args: true,
11951 /* AllocationBlock*/ args&: OuterAllocBB,
11952 /* DeallocationBlocks */ args: ArrayRef<BasicBlock *>(),
11953 /* Suffix */ args: Suffix.str(), args&: ArgsInZeroAddressSpace);
11954}
11955
11956std::unique_ptr<CodeExtractor> DeviceSharedMemOutlineInfo::createCodeExtractor(
11957 ArrayRef<BasicBlock *> Blocks, bool ArgsInZeroAddressSpace, Twine Suffix) {
11958 return std::make_unique<DeviceSharedMemCodeExtractor>(
11959 args&: OMPBuilder, args&: Blocks, /* DominatorTree */ args: nullptr,
11960 /* AggregateArgs */ args: true,
11961 /* BlockFrequencyInfo */ args: nullptr,
11962 /* BranchProbabilityInfo */ args: nullptr,
11963 /* AssumptionCache */ args: nullptr,
11964 /* AllowVarArgs */ args: true,
11965 /* AllowAlloca */ args: true,
11966 /* AllocationBlock*/ args&: OuterAllocBB,
11967 /* DeallocationBlocks */ args: OuterDeallocBBs.empty()
11968 ? SmallVector<BasicBlock *>{ExitBB}
11969 : OuterDeallocBBs,
11970 /* Suffix */ args: Suffix.str(), args&: ArgsInZeroAddressSpace);
11971}
11972
11973void OpenMPIRBuilder::createOffloadEntry(Constant *ID, Constant *Addr,
11974 uint64_t Size, int32_t Flags,
11975 GlobalValue::LinkageTypes,
11976 StringRef Name) {
11977 if (!Config.isGPU()) {
11978 llvm::offloading::emitOffloadingEntry(
11979 M, Kind: object::OffloadKind::OFK_OpenMP, Addr: ID,
11980 Name: Name.empty() ? Addr->getName() : Name, Size, Flags, /*Data=*/0);
11981 return;
11982 }
11983 // TODO: Add support for global variables on the device after declare target
11984 // support.
11985 Function *Fn = dyn_cast<Function>(Val: Addr);
11986 if (!Fn)
11987 return;
11988
11989 // Add a function attribute for the kernel.
11990 Fn->addFnAttr(Kind: "kernel");
11991 if (T.isAMDGCN())
11992 Fn->addFnAttr(Kind: "uniform-work-group-size");
11993 Fn->addFnAttr(Kind: Attribute::MustProgress);
11994}
11995
11996// We only generate metadata for function that contain target regions.
11997void OpenMPIRBuilder::createOffloadEntriesAndInfoMetadata(
11998 EmitMetadataErrorReportFunctionTy &ErrorFn) {
11999
12000 // If there are no entries, we don't need to do anything.
12001 if (OffloadInfoManager.empty())
12002 return;
12003
12004 LLVMContext &C = M.getContext();
12005 SmallVector<std::pair<const OffloadEntriesInfoManager::OffloadEntryInfo *,
12006 TargetRegionEntryInfo>,
12007 16>
12008 OrderedEntries(OffloadInfoManager.size());
12009
12010 // Auxiliary methods to create metadata values and strings.
12011 auto &&GetMDInt = [this](unsigned V) {
12012 return ConstantAsMetadata::get(C: ConstantInt::get(Ty: Builder.getInt32Ty(), V));
12013 };
12014
12015 auto &&GetMDString = [&C](StringRef V) { return MDString::get(Context&: C, Str: V); };
12016
12017 // Create the offloading info metadata node.
12018 NamedMDNode *MD = M.getOrInsertNamedMetadata(Name: "omp_offload.info");
12019 auto &&TargetRegionMetadataEmitter =
12020 [&C, MD, &OrderedEntries, &GetMDInt, &GetMDString](
12021 const TargetRegionEntryInfo &EntryInfo,
12022 const OffloadEntriesInfoManager::OffloadEntryInfoTargetRegion &E) {
12023 // Generate metadata for target regions. Each entry of this metadata
12024 // contains:
12025 // - Entry 0 -> Kind of this type of metadata (0).
12026 // - Entry 1 -> Device ID of the file where the entry was identified.
12027 // - Entry 2 -> File ID of the file where the entry was identified.
12028 // - Entry 3 -> Mangled name of the function where the entry was
12029 // identified.
12030 // - Entry 4 -> Line in the file where the entry was identified.
12031 // - Entry 5 -> Count of regions at this DeviceID/FilesID/Line.
12032 // - Entry 6 -> Order the entry was created.
12033 // The first element of the metadata node is the kind.
12034 Metadata *Ops[] = {
12035 GetMDInt(E.getKind()), GetMDInt(EntryInfo.DeviceID),
12036 GetMDInt(EntryInfo.FileID), GetMDString(EntryInfo.ParentName),
12037 GetMDInt(EntryInfo.Line), GetMDInt(EntryInfo.Count),
12038 GetMDInt(E.getOrder())};
12039
12040 // Save this entry in the right position of the ordered entries array.
12041 OrderedEntries[E.getOrder()] = std::make_pair(x: &E, y: EntryInfo);
12042
12043 // Add metadata to the named metadata node.
12044 MD->addOperand(M: MDNode::get(Context&: C, MDs: Ops));
12045 };
12046
12047 OffloadInfoManager.actOnTargetRegionEntriesInfo(Action: TargetRegionMetadataEmitter);
12048
12049 // Create function that emits metadata for each device global variable entry;
12050 auto &&DeviceGlobalVarMetadataEmitter =
12051 [&C, &OrderedEntries, &GetMDInt, &GetMDString, MD](
12052 StringRef MangledName,
12053 const OffloadEntriesInfoManager::OffloadEntryInfoDeviceGlobalVar &E) {
12054 // Generate metadata for global variables. Each entry of this metadata
12055 // contains:
12056 // - Entry 0 -> Kind of this type of metadata (1).
12057 // - Entry 1 -> Mangled name of the variable.
12058 // - Entry 2 -> Declare target kind.
12059 // - Entry 3 -> Order the entry was created.
12060 // The first element of the metadata node is the kind.
12061 Metadata *Ops[] = {GetMDInt(E.getKind()), GetMDString(MangledName),
12062 GetMDInt(E.getFlags()), GetMDInt(E.getOrder())};
12063
12064 // Save this entry in the right position of the ordered entries array.
12065 TargetRegionEntryInfo varInfo(MangledName, 0, 0, 0);
12066 OrderedEntries[E.getOrder()] = std::make_pair(x: &E, y&: varInfo);
12067
12068 // Add metadata to the named metadata node.
12069 MD->addOperand(M: MDNode::get(Context&: C, MDs: Ops));
12070 };
12071
12072 OffloadInfoManager.actOnDeviceGlobalVarEntriesInfo(
12073 Action: DeviceGlobalVarMetadataEmitter);
12074
12075 for (const auto &E : OrderedEntries) {
12076 assert(E.first && "All ordered entries must exist!");
12077 if (const auto *CE =
12078 dyn_cast<OffloadEntriesInfoManager::OffloadEntryInfoTargetRegion>(
12079 Val: E.first)) {
12080 if (!CE->getID() || !CE->getAddress()) {
12081 // Do not blame the entry if the parent funtion is not emitted.
12082 TargetRegionEntryInfo EntryInfo = E.second;
12083 StringRef FnName = EntryInfo.ParentName;
12084 if (!M.getNamedValue(Name: FnName))
12085 continue;
12086 ErrorFn(EMIT_MD_TARGET_REGION_ERROR, EntryInfo);
12087 continue;
12088 }
12089 createOffloadEntry(ID: CE->getID(), Addr: CE->getAddress(),
12090 /*Size=*/0, Flags: CE->getFlags(),
12091 GlobalValue::WeakAnyLinkage);
12092 } else if (const auto *CE = dyn_cast<
12093 OffloadEntriesInfoManager::OffloadEntryInfoDeviceGlobalVar>(
12094 Val: E.first)) {
12095 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind Flags =
12096 static_cast<OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind>(
12097 CE->getFlags());
12098 switch (Flags) {
12099 case OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter:
12100 case OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo:
12101 if (Config.isTargetDevice() && Config.hasRequiresUnifiedSharedMemory())
12102 continue;
12103 if (!CE->getAddress()) {
12104 ErrorFn(EMIT_MD_DECLARE_TARGET_ERROR, E.second);
12105 continue;
12106 }
12107 // The vaiable has no definition - no need to add the entry.
12108 if (CE->getVarSize() == 0)
12109 continue;
12110 break;
12111 case OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink:
12112 assert(((Config.isTargetDevice() && !CE->getAddress()) ||
12113 (!Config.isTargetDevice() && CE->getAddress())) &&
12114 "Declaret target link address is set.");
12115 if (Config.isTargetDevice())
12116 continue;
12117 if (!CE->getAddress()) {
12118 ErrorFn(EMIT_MD_GLOBAL_VAR_LINK_ERROR, TargetRegionEntryInfo());
12119 continue;
12120 }
12121 break;
12122 case OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect:
12123 case OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirectVTable:
12124 if (!CE->getAddress()) {
12125 ErrorFn(EMIT_MD_GLOBAL_VAR_INDIRECT_ERROR, E.second);
12126 continue;
12127 }
12128 break;
12129 default:
12130 break;
12131 }
12132
12133 // Hidden or internal symbols on the device are not externally visible.
12134 // We should not attempt to register them by creating an offloading
12135 // entry. Indirect variables are handled separately on the device.
12136 if (auto *GV = dyn_cast<GlobalValue>(Val: CE->getAddress()))
12137 if ((GV->hasLocalLinkage() || GV->hasHiddenVisibility()) &&
12138 (Flags !=
12139 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect &&
12140 Flags != OffloadEntriesInfoManager::
12141 OMPTargetGlobalVarEntryIndirectVTable))
12142 continue;
12143
12144 // Indirect globals need to use a special name that doesn't match the name
12145 // of the associated host global.
12146 if (Flags == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect ||
12147 Flags ==
12148 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirectVTable)
12149 createOffloadEntry(ID: CE->getAddress(), Addr: CE->getAddress(), Size: CE->getVarSize(),
12150 Flags, CE->getLinkage(), Name: CE->getVarName());
12151 else
12152 createOffloadEntry(ID: CE->getAddress(), Addr: CE->getAddress(), Size: CE->getVarSize(),
12153 Flags, CE->getLinkage());
12154
12155 } else {
12156 llvm_unreachable("Unsupported entry kind.");
12157 }
12158 }
12159
12160 // Emit requires directive globals to a special entry so the runtime can
12161 // register them when the device image is loaded.
12162 // TODO: This reduces the offloading entries to a 32-bit integer. Offloading
12163 // entries should be redesigned to better suit this use-case.
12164 if (Config.hasRequiresFlags() && !Config.isTargetDevice())
12165 offloading::emitOffloadingEntry(
12166 M, Kind: object::OffloadKind::OFK_OpenMP,
12167 Addr: Constant::getNullValue(Ty: PointerType::getUnqual(C&: M.getContext())),
12168 Name: ".requires", /*Size=*/0,
12169 Flags: OffloadEntriesInfoManager::OMPTargetGlobalRegisterRequires,
12170 Data: Config.getRequiresFlags());
12171}
12172
12173void TargetRegionEntryInfo::getTargetRegionEntryFnName(
12174 SmallVectorImpl<char> &Name, StringRef ParentName, unsigned DeviceID,
12175 unsigned FileID, unsigned Line, unsigned Count) {
12176 raw_svector_ostream OS(Name);
12177 OS << KernelNamePrefix << llvm::format(Fmt: "%x", Vals: DeviceID)
12178 << llvm::format(Fmt: "_%x_", Vals: FileID) << ParentName << "_l" << Line;
12179 if (Count)
12180 OS << "_" << Count;
12181}
12182
12183void OffloadEntriesInfoManager::getTargetRegionEntryFnName(
12184 SmallVectorImpl<char> &Name, const TargetRegionEntryInfo &EntryInfo) {
12185 unsigned NewCount = getTargetRegionEntryInfoCount(EntryInfo);
12186 TargetRegionEntryInfo::getTargetRegionEntryFnName(
12187 Name, ParentName: EntryInfo.ParentName, DeviceID: EntryInfo.DeviceID, FileID: EntryInfo.FileID,
12188 Line: EntryInfo.Line, Count: NewCount);
12189}
12190
12191TargetRegionEntryInfo
12192OpenMPIRBuilder::getTargetEntryUniqueInfo(FileIdentifierInfoCallbackTy CallBack,
12193 vfs::FileSystem &VFS,
12194 StringRef ParentName) {
12195 sys::fs::UniqueID ID(0xdeadf17e, 0);
12196 auto FileIDInfo = CallBack();
12197 uint64_t FileID = 0;
12198 if (ErrorOr<vfs::Status> Status = VFS.status(Path: std::get<0>(t&: FileIDInfo))) {
12199 ID = Status->getUniqueID();
12200 FileID = Status->getUniqueID().getFile();
12201 } else {
12202 // If the inode ID could not be determined, create a hash value
12203 // the current file name and use that as an ID.
12204 FileID = hash_value(arg: std::get<0>(t&: FileIDInfo));
12205 }
12206
12207 return TargetRegionEntryInfo(ParentName, ID.getDevice(), FileID,
12208 std::get<1>(t&: FileIDInfo));
12209}
12210
12211unsigned OpenMPIRBuilder::getFlagMemberOffset() {
12212 unsigned Offset = 0;
12213 for (uint64_t Remain =
12214 static_cast<std::underlying_type_t<omp::OpenMPOffloadMappingFlags>>(
12215 omp::OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);
12216 !(Remain & 1); Remain = Remain >> 1)
12217 Offset++;
12218 return Offset;
12219}
12220
12221omp::OpenMPOffloadMappingFlags
12222OpenMPIRBuilder::getMemberOfFlag(unsigned Position) {
12223 // Rotate by getFlagMemberOffset() bits.
12224 return static_cast<omp::OpenMPOffloadMappingFlags>(((uint64_t)Position + 1)
12225 << getFlagMemberOffset());
12226}
12227
12228void OpenMPIRBuilder::setCorrectMemberOfFlag(
12229 omp::OpenMPOffloadMappingFlags &Flags,
12230 omp::OpenMPOffloadMappingFlags MemberOfFlag) {
12231 // If the entry is PTR_AND_OBJ but has not been marked with the special
12232 // placeholder value 0xFFFF in the MEMBER_OF field, then it should not be
12233 // marked as MEMBER_OF.
12234 if (static_cast<std::underlying_type_t<omp::OpenMPOffloadMappingFlags>>(
12235 Flags & omp::OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ) &&
12236 static_cast<std::underlying_type_t<omp::OpenMPOffloadMappingFlags>>(
12237 (Flags & omp::OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) !=
12238 omp::OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF))
12239 return;
12240
12241 // Entries with ATTACH are not members-of anything. They are handled
12242 // separately by the runtime after other maps have been handled.
12243 if (static_cast<std::underlying_type_t<omp::OpenMPOffloadMappingFlags>>(
12244 Flags & omp::OpenMPOffloadMappingFlags::OMP_MAP_ATTACH))
12245 return;
12246
12247 // Reset the placeholder value to prepare the flag for the assignment of the
12248 // proper MEMBER_OF value.
12249 Flags &= ~omp::OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF;
12250 Flags |= MemberOfFlag;
12251}
12252
12253Constant *OpenMPIRBuilder::getAddrOfDeclareTargetVar(
12254 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind CaptureClause,
12255 OffloadEntriesInfoManager::OMPTargetDeviceClauseKind DeviceClause,
12256 bool IsDeclaration, bool IsExternallyVisible,
12257 TargetRegionEntryInfo EntryInfo, StringRef MangledName,
12258 std::vector<GlobalVariable *> &GeneratedRefs, bool OpenMPSIMD,
12259 std::vector<Triple> TargetTriple, Type *LlvmPtrTy,
12260 std::function<Constant *()> GlobalInitializer,
12261 std::function<GlobalValue::LinkageTypes()> VariableLinkage) {
12262 // TODO: convert this to utilise the IRBuilder Config rather than
12263 // a passed down argument.
12264 if (OpenMPSIMD)
12265 return nullptr;
12266
12267 if (CaptureClause == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink ||
12268 ((CaptureClause == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo ||
12269 CaptureClause ==
12270 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter) &&
12271 Config.hasRequiresUnifiedSharedMemory())) {
12272 SmallString<64> PtrName;
12273 {
12274 raw_svector_ostream OS(PtrName);
12275 OS << MangledName;
12276 if (!IsExternallyVisible)
12277 OS << format(Fmt: "_%x", Vals: EntryInfo.FileID);
12278 OS << "_decl_tgt_ref_ptr";
12279 }
12280
12281 Value *Ptr = M.getNamedValue(Name: PtrName);
12282
12283 if (!Ptr) {
12284 GlobalValue *GlobalValue = M.getNamedValue(Name: MangledName);
12285 Ptr = getOrCreateInternalVariable(Ty: LlvmPtrTy, Name: PtrName);
12286
12287 auto *GV = cast<GlobalVariable>(Val: Ptr);
12288 GV->setLinkage(GlobalValue::WeakAnyLinkage);
12289
12290 if (!Config.isTargetDevice()) {
12291 if (GlobalInitializer)
12292 GV->setInitializer(GlobalInitializer());
12293 else
12294 GV->setInitializer(GlobalValue);
12295 }
12296
12297 registerTargetGlobalVariable(
12298 CaptureClause, DeviceClause, IsDeclaration, IsExternallyVisible,
12299 EntryInfo, MangledName, GeneratedRefs, OpenMPSIMD, TargetTriple,
12300 GlobalInitializer, VariableLinkage, LlvmPtrTy, Addr: cast<Constant>(Val: Ptr));
12301 }
12302
12303 return cast<Constant>(Val: Ptr);
12304 }
12305
12306 return nullptr;
12307}
12308
12309void OpenMPIRBuilder::registerTargetGlobalVariable(
12310 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind CaptureClause,
12311 OffloadEntriesInfoManager::OMPTargetDeviceClauseKind DeviceClause,
12312 bool IsDeclaration, bool IsExternallyVisible,
12313 TargetRegionEntryInfo EntryInfo, StringRef MangledName,
12314 std::vector<GlobalVariable *> &GeneratedRefs, bool OpenMPSIMD,
12315 std::vector<Triple> TargetTriple,
12316 std::function<Constant *()> GlobalInitializer,
12317 std::function<GlobalValue::LinkageTypes()> VariableLinkage, Type *LlvmPtrTy,
12318 Constant *Addr) {
12319 if (DeviceClause != OffloadEntriesInfoManager::OMPTargetDeviceClauseAny ||
12320 (TargetTriple.empty() && !Config.isTargetDevice()))
12321 return;
12322
12323 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind Flags;
12324 StringRef VarName;
12325 int64_t VarSize;
12326 GlobalValue::LinkageTypes Linkage;
12327
12328 if ((CaptureClause == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo ||
12329 CaptureClause ==
12330 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter) &&
12331 !Config.hasRequiresUnifiedSharedMemory()) {
12332 Flags = OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo;
12333 VarName = MangledName;
12334 GlobalValue *LlvmVal = M.getNamedValue(Name: VarName);
12335
12336 if (!IsDeclaration)
12337 VarSize = divideCeil(
12338 Numerator: M.getDataLayout().getTypeSizeInBits(Ty: LlvmVal->getValueType()), Denominator: 8);
12339 else
12340 VarSize = 0;
12341 Linkage = (VariableLinkage) ? VariableLinkage() : LlvmVal->getLinkage();
12342
12343 // This is a workaround carried over from Clang which prevents undesired
12344 // optimisation of internal variables.
12345 if (Config.isTargetDevice() &&
12346 (!IsExternallyVisible || Linkage == GlobalValue::LinkOnceODRLinkage)) {
12347 // Do not create a "ref-variable" if the original is not also available
12348 // on the host.
12349 if (!OffloadInfoManager.hasDeviceGlobalVarEntryInfo(VarName))
12350 return;
12351
12352 std::string RefName = createPlatformSpecificName(Parts: {VarName, "ref"});
12353
12354 if (!M.getNamedValue(Name: RefName)) {
12355 Constant *AddrRef =
12356 getOrCreateInternalVariable(Ty: Addr->getType(), Name: RefName);
12357 auto *GvAddrRef = cast<GlobalVariable>(Val: AddrRef);
12358 GvAddrRef->setConstant(true);
12359 GvAddrRef->setLinkage(GlobalValue::InternalLinkage);
12360 GvAddrRef->setInitializer(Addr);
12361 GeneratedRefs.push_back(x: GvAddrRef);
12362 }
12363 }
12364 } else {
12365 if (CaptureClause == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink)
12366 Flags = OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink;
12367 else
12368 Flags = OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo;
12369
12370 if (Config.isTargetDevice()) {
12371 VarName = (Addr) ? Addr->getName() : "";
12372 Addr = nullptr;
12373 } else {
12374 Addr = getAddrOfDeclareTargetVar(
12375 CaptureClause, DeviceClause, IsDeclaration, IsExternallyVisible,
12376 EntryInfo, MangledName, GeneratedRefs, OpenMPSIMD, TargetTriple,
12377 LlvmPtrTy, GlobalInitializer, VariableLinkage);
12378 VarName = (Addr) ? Addr->getName() : "";
12379 }
12380 VarSize = M.getDataLayout().getPointerSize();
12381 Linkage = GlobalValue::WeakAnyLinkage;
12382 }
12383
12384 OffloadInfoManager.registerDeviceGlobalVarEntryInfo(VarName, Addr, VarSize,
12385 Flags, Linkage);
12386}
12387
12388/// Loads all the offload entries information from the host IR
12389/// metadata.
12390void OpenMPIRBuilder::loadOffloadInfoMetadata(Module &M) {
12391 // If we are in target mode, load the metadata from the host IR. This code has
12392 // to match the metadata creation in createOffloadEntriesAndInfoMetadata().
12393
12394 NamedMDNode *MD = M.getNamedMetadata(Name: ompOffloadInfoName);
12395 if (!MD)
12396 return;
12397
12398 for (MDNode *MN : MD->operands()) {
12399 auto &&GetMDInt = [MN](unsigned Idx) {
12400 auto *V = cast<ConstantAsMetadata>(Val: MN->getOperand(I: Idx));
12401 return cast<ConstantInt>(Val: V->getValue())->getZExtValue();
12402 };
12403
12404 auto &&GetMDString = [MN](unsigned Idx) {
12405 auto *V = cast<MDString>(Val: MN->getOperand(I: Idx));
12406 return V->getString();
12407 };
12408
12409 switch (GetMDInt(0)) {
12410 default:
12411 llvm_unreachable("Unexpected metadata!");
12412 break;
12413 case OffloadEntriesInfoManager::OffloadEntryInfo::
12414 OffloadingEntryInfoTargetRegion: {
12415 TargetRegionEntryInfo EntryInfo(/*ParentName=*/GetMDString(3),
12416 /*DeviceID=*/GetMDInt(1),
12417 /*FileID=*/GetMDInt(2),
12418 /*Line=*/GetMDInt(4),
12419 /*Count=*/GetMDInt(5));
12420 OffloadInfoManager.initializeTargetRegionEntryInfo(EntryInfo,
12421 /*Order=*/GetMDInt(6));
12422 break;
12423 }
12424 case OffloadEntriesInfoManager::OffloadEntryInfo::
12425 OffloadingEntryInfoDeviceGlobalVar:
12426 OffloadInfoManager.initializeDeviceGlobalVarEntryInfo(
12427 /*MangledName=*/Name: GetMDString(1),
12428 Flags: static_cast<OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind>(
12429 /*Flags=*/GetMDInt(2)),
12430 /*Order=*/GetMDInt(3));
12431 break;
12432 }
12433 }
12434}
12435
12436void OpenMPIRBuilder::loadOffloadInfoMetadata(vfs::FileSystem &VFS,
12437 StringRef HostFilePath) {
12438 if (HostFilePath.empty())
12439 return;
12440
12441 auto Buf = VFS.getBufferForFile(Name: HostFilePath);
12442 if (std::error_code Err = Buf.getError()) {
12443 report_fatal_error(reason: ("error opening host file from host file path inside of "
12444 "OpenMPIRBuilder: " +
12445 Err.message())
12446 .c_str());
12447 }
12448
12449 LLVMContext Ctx;
12450 auto M = expectedToErrorOrAndEmitErrors(
12451 Ctx, Val: parseBitcodeFile(Buffer: Buf.get()->getMemBufferRef(), Context&: Ctx));
12452 if (std::error_code Err = M.getError()) {
12453 report_fatal_error(
12454 reason: ("error parsing host file inside of OpenMPIRBuilder: " + Err.message())
12455 .c_str());
12456 }
12457
12458 loadOffloadInfoMetadata(M&: *M.get());
12459}
12460
12461OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createIteratorLoop(
12462 LocationDescription Loc, llvm::Value *TripCount, IteratorBodyGenTy BodyGen,
12463 llvm::StringRef Name) {
12464 Builder.restoreIP(IP: Loc.IP);
12465
12466 BasicBlock *CurBB = Builder.GetInsertBlock();
12467 assert(CurBB &&
12468 "expected a valid insertion block for creating an iterator loop");
12469 Function *F = CurBB->getParent();
12470
12471 InsertPointTy SplitIP = Builder.saveIP();
12472 if (SplitIP.getPoint() == CurBB->end())
12473 if (Instruction *Terminator = CurBB->getTerminatorOrNull())
12474 SplitIP = InsertPointTy(CurBB, Terminator->getIterator());
12475
12476 BasicBlock *ContBB =
12477 splitBB(IP: SplitIP, /*CreateBranch=*/false,
12478 DL: Builder.getCurrentDebugLocation(), Name: "omp.it.cont");
12479
12480 CanonicalLoopInfo *CLI =
12481 createLoopSkeleton(DL: Builder.getCurrentDebugLocation(), TripCount, F,
12482 /*PreInsertBefore=*/ContBB,
12483 /*PostInsertBefore=*/ContBB, Name);
12484
12485 // Enter loop from original block.
12486 redirectTo(Source: CurBB, Target: CLI->getPreheader(), DL: Builder.getCurrentDebugLocation());
12487
12488 // Remove the unconditional branch inserted by createLoopSkeleton in the body
12489 if (Instruction *T = CLI->getBody()->getTerminatorOrNull())
12490 T->eraseFromParent();
12491
12492 InsertPointTy BodyIP = CLI->getBodyIP();
12493 if (llvm::Error Err = BodyGen(BodyIP, CLI->getIndVar()))
12494 return Err;
12495
12496 // Body must either fallthrough to the latch or branch directly to it.
12497 if (Instruction *BodyTerminator = CLI->getBody()->getTerminatorOrNull()) {
12498 auto *BodyBr = dyn_cast<UncondBrInst>(Val: BodyTerminator);
12499 if (!BodyBr || BodyBr->getSuccessor() != CLI->getLatch()) {
12500 return make_error<StringError>(
12501 Args: "iterator bodygen must terminate the canonical body with an "
12502 "unconditional branch to the loop latch",
12503 Args: inconvertibleErrorCode());
12504 }
12505 } else {
12506 // Ensure we end the loop body by jumping to the latch.
12507 Builder.SetInsertPoint(CLI->getBody());
12508 Builder.CreateBr(Dest: CLI->getLatch());
12509 }
12510
12511 // Link After -> ContBB
12512 Builder.SetInsertPoint(TheBB: CLI->getAfter(), IP: CLI->getAfter()->begin());
12513 if (!CLI->getAfter()->hasTerminator())
12514 Builder.CreateBr(Dest: ContBB);
12515
12516 return InsertPointTy{ContBB, ContBB->begin()};
12517}
12518
12519/// Mangle the parameter part of the vector function name according to
12520/// their OpenMP classification. The mangling function is defined in
12521/// section 4.5 of the AAVFABI(2021Q1).
12522static std::string mangleVectorParameters(
12523 ArrayRef<llvm::OpenMPIRBuilder::DeclareSimdAttrTy> ParamAttrs) {
12524 SmallString<256> Buffer;
12525 llvm::raw_svector_ostream Out(Buffer);
12526 for (const auto &ParamAttr : ParamAttrs) {
12527 switch (ParamAttr.Kind) {
12528 case llvm::OpenMPIRBuilder::DeclareSimdKindTy::Linear:
12529 Out << 'l';
12530 break;
12531 case llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearRef:
12532 Out << 'R';
12533 break;
12534 case llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearUVal:
12535 Out << 'U';
12536 break;
12537 case llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearVal:
12538 Out << 'L';
12539 break;
12540 case llvm::OpenMPIRBuilder::DeclareSimdKindTy::Uniform:
12541 Out << 'u';
12542 break;
12543 case llvm::OpenMPIRBuilder::DeclareSimdKindTy::Vector:
12544 Out << 'v';
12545 break;
12546 }
12547 if (ParamAttr.HasVarStride)
12548 Out << "s" << ParamAttr.StrideOrArg;
12549 else if (ParamAttr.Kind ==
12550 llvm::OpenMPIRBuilder::DeclareSimdKindTy::Linear ||
12551 ParamAttr.Kind ==
12552 llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearRef ||
12553 ParamAttr.Kind ==
12554 llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearUVal ||
12555 ParamAttr.Kind ==
12556 llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearVal) {
12557 // Don't print the step value if it is not present or if it is
12558 // equal to 1.
12559 if (ParamAttr.StrideOrArg < 0)
12560 Out << 'n' << -ParamAttr.StrideOrArg;
12561 else if (ParamAttr.StrideOrArg != 1)
12562 Out << ParamAttr.StrideOrArg;
12563 }
12564
12565 if (!!ParamAttr.Alignment)
12566 Out << 'a' << ParamAttr.Alignment;
12567 }
12568
12569 return std::string(Out.str());
12570}
12571
12572void OpenMPIRBuilder::emitX86DeclareSimdFunction(
12573 llvm::Function *Fn, unsigned NumElts, const llvm::APSInt &VLENVal,
12574 llvm::ArrayRef<DeclareSimdAttrTy> ParamAttrs, DeclareSimdBranch Branch) {
12575 struct ISADataTy {
12576 char ISA;
12577 unsigned VecRegSize;
12578 };
12579 ISADataTy ISAData[] = {
12580 {.ISA: 'b', .VecRegSize: 128}, // SSE
12581 {.ISA: 'c', .VecRegSize: 256}, // AVX
12582 {.ISA: 'd', .VecRegSize: 256}, // AVX2
12583 {.ISA: 'e', .VecRegSize: 512}, // AVX512
12584 };
12585 llvm::SmallVector<char, 2> Masked;
12586 switch (Branch) {
12587 case DeclareSimdBranch::Undefined:
12588 Masked.push_back(Elt: 'N');
12589 Masked.push_back(Elt: 'M');
12590 break;
12591 case DeclareSimdBranch::Notinbranch:
12592 Masked.push_back(Elt: 'N');
12593 break;
12594 case DeclareSimdBranch::Inbranch:
12595 Masked.push_back(Elt: 'M');
12596 break;
12597 }
12598 for (char Mask : Masked) {
12599 for (const ISADataTy &Data : ISAData) {
12600 llvm::SmallString<256> Buffer;
12601 llvm::raw_svector_ostream Out(Buffer);
12602 Out << "_ZGV" << Data.ISA << Mask;
12603 if (!VLENVal) {
12604 assert(NumElts && "Non-zero simdlen/cdtsize expected");
12605 Out << llvm::APSInt::getUnsigned(X: Data.VecRegSize / NumElts);
12606 } else {
12607 Out << VLENVal;
12608 }
12609 Out << mangleVectorParameters(ParamAttrs);
12610 Out << '_' << Fn->getName();
12611 Fn->addFnAttr(Kind: Out.str());
12612 }
12613 }
12614}
12615
12616// Function used to add the attribute. The parameter `VLEN` is templated to
12617// allow the use of `x` when targeting scalable functions for SVE.
12618template <typename T>
12619static void addAArch64VectorName(T VLEN, StringRef LMask, StringRef Prefix,
12620 char ISA, StringRef ParSeq,
12621 StringRef MangledName, bool OutputBecomesInput,
12622 llvm::Function *Fn) {
12623 SmallString<256> Buffer;
12624 llvm::raw_svector_ostream Out(Buffer);
12625 Out << Prefix << ISA << LMask << VLEN;
12626 if (OutputBecomesInput)
12627 Out << 'v';
12628 Out << ParSeq << '_' << MangledName;
12629 Fn->addFnAttr(Kind: Out.str());
12630}
12631
12632// Helper function to generate the Advanced SIMD names depending on the value
12633// of the NDS when simdlen is not present.
12634static void addAArch64AdvSIMDNDSNames(unsigned NDS, StringRef Mask,
12635 StringRef Prefix, char ISA,
12636 StringRef ParSeq, StringRef MangledName,
12637 bool OutputBecomesInput,
12638 llvm::Function *Fn) {
12639 switch (NDS) {
12640 case 8:
12641 addAArch64VectorName(VLEN: 8, LMask: Mask, Prefix, ISA, ParSeq, MangledName,
12642 OutputBecomesInput, Fn);
12643 addAArch64VectorName(VLEN: 16, LMask: Mask, Prefix, ISA, ParSeq, MangledName,
12644 OutputBecomesInput, Fn);
12645 break;
12646 case 16:
12647 addAArch64VectorName(VLEN: 4, LMask: Mask, Prefix, ISA, ParSeq, MangledName,
12648 OutputBecomesInput, Fn);
12649 addAArch64VectorName(VLEN: 8, LMask: Mask, Prefix, ISA, ParSeq, MangledName,
12650 OutputBecomesInput, Fn);
12651 break;
12652 case 32:
12653 addAArch64VectorName(VLEN: 2, LMask: Mask, Prefix, ISA, ParSeq, MangledName,
12654 OutputBecomesInput, Fn);
12655 addAArch64VectorName(VLEN: 4, LMask: Mask, Prefix, ISA, ParSeq, MangledName,
12656 OutputBecomesInput, Fn);
12657 break;
12658 case 64:
12659 case 128:
12660 addAArch64VectorName(VLEN: 2, LMask: Mask, Prefix, ISA, ParSeq, MangledName,
12661 OutputBecomesInput, Fn);
12662 break;
12663 default:
12664 llvm_unreachable("Scalar type is too wide.");
12665 }
12666}
12667
12668/// Emit vector function attributes for AArch64, as defined in the AAVFABI.
12669void OpenMPIRBuilder::emitAArch64DeclareSimdFunction(
12670 llvm::Function *Fn, unsigned UserVLEN,
12671 llvm::ArrayRef<DeclareSimdAttrTy> ParamAttrs, DeclareSimdBranch Branch,
12672 char ISA, unsigned NarrowestDataSize, bool OutputBecomesInput) {
12673 assert((ISA == 'n' || ISA == 's') && "Expected ISA either 's' or 'n'.");
12674
12675 // Sort out parameter sequence.
12676 const std::string ParSeq = mangleVectorParameters(ParamAttrs);
12677 StringRef Prefix = "_ZGV";
12678 StringRef MangledName = Fn->getName();
12679
12680 // Generate simdlen from user input (if any).
12681 if (UserVLEN) {
12682 if (ISA == 's') {
12683 // SVE generates only a masked function.
12684 addAArch64VectorName(VLEN: UserVLEN, LMask: "M", Prefix, ISA, ParSeq, MangledName,
12685 OutputBecomesInput, Fn);
12686 return;
12687 }
12688
12689 switch (Branch) {
12690 case DeclareSimdBranch::Undefined:
12691 addAArch64VectorName(VLEN: UserVLEN, LMask: "N", Prefix, ISA, ParSeq, MangledName,
12692 OutputBecomesInput, Fn);
12693 addAArch64VectorName(VLEN: UserVLEN, LMask: "M", Prefix, ISA, ParSeq, MangledName,
12694 OutputBecomesInput, Fn);
12695 break;
12696 case DeclareSimdBranch::Inbranch:
12697 addAArch64VectorName(VLEN: UserVLEN, LMask: "M", Prefix, ISA, ParSeq, MangledName,
12698 OutputBecomesInput, Fn);
12699 break;
12700 case DeclareSimdBranch::Notinbranch:
12701 addAArch64VectorName(VLEN: UserVLEN, LMask: "N", Prefix, ISA, ParSeq, MangledName,
12702 OutputBecomesInput, Fn);
12703 break;
12704 }
12705 return;
12706 }
12707
12708 if (ISA == 's') {
12709 // SVE, section 3.4.1, item 1.
12710 addAArch64VectorName(VLEN: "x", LMask: "M", Prefix, ISA, ParSeq, MangledName,
12711 OutputBecomesInput, Fn);
12712 return;
12713 }
12714
12715 switch (Branch) {
12716 case DeclareSimdBranch::Undefined:
12717 addAArch64AdvSIMDNDSNames(NDS: NarrowestDataSize, Mask: "N", Prefix, ISA, ParSeq,
12718 MangledName, OutputBecomesInput, Fn);
12719 addAArch64AdvSIMDNDSNames(NDS: NarrowestDataSize, Mask: "M", Prefix, ISA, ParSeq,
12720 MangledName, OutputBecomesInput, Fn);
12721 break;
12722 case DeclareSimdBranch::Inbranch:
12723 addAArch64AdvSIMDNDSNames(NDS: NarrowestDataSize, Mask: "M", Prefix, ISA, ParSeq,
12724 MangledName, OutputBecomesInput, Fn);
12725 break;
12726 case DeclareSimdBranch::Notinbranch:
12727 addAArch64AdvSIMDNDSNames(NDS: NarrowestDataSize, Mask: "N", Prefix, ISA, ParSeq,
12728 MangledName, OutputBecomesInput, Fn);
12729 break;
12730 }
12731}
12732
12733//===----------------------------------------------------------------------===//
12734// OffloadEntriesInfoManager
12735//===----------------------------------------------------------------------===//
12736
12737bool OffloadEntriesInfoManager::empty() const {
12738 return OffloadEntriesTargetRegion.empty() &&
12739 OffloadEntriesDeviceGlobalVar.empty();
12740}
12741
12742unsigned OffloadEntriesInfoManager::getTargetRegionEntryInfoCount(
12743 const TargetRegionEntryInfo &EntryInfo) const {
12744 auto It = OffloadEntriesTargetRegionCount.find(
12745 x: getTargetRegionEntryCountKey(EntryInfo));
12746 if (It == OffloadEntriesTargetRegionCount.end())
12747 return 0;
12748 return It->second;
12749}
12750
12751void OffloadEntriesInfoManager::incrementTargetRegionEntryInfoCount(
12752 const TargetRegionEntryInfo &EntryInfo) {
12753 OffloadEntriesTargetRegionCount[getTargetRegionEntryCountKey(EntryInfo)] =
12754 EntryInfo.Count + 1;
12755}
12756
12757/// Initialize target region entry.
12758void OffloadEntriesInfoManager::initializeTargetRegionEntryInfo(
12759 const TargetRegionEntryInfo &EntryInfo, unsigned Order) {
12760 OffloadEntriesTargetRegion[EntryInfo] =
12761 OffloadEntryInfoTargetRegion(Order, /*Addr=*/nullptr, /*ID=*/nullptr,
12762 OMPTargetRegionEntryTargetRegion);
12763 ++OffloadingEntriesNum;
12764}
12765
12766void OffloadEntriesInfoManager::registerTargetRegionEntryInfo(
12767 TargetRegionEntryInfo EntryInfo, Constant *Addr, Constant *ID,
12768 OMPTargetRegionEntryKind Flags) {
12769 assert(EntryInfo.Count == 0 && "expected default EntryInfo");
12770
12771 // Update the EntryInfo with the next available count for this location.
12772 EntryInfo.Count = getTargetRegionEntryInfoCount(EntryInfo);
12773
12774 // If we are emitting code for a target, the entry is already initialized,
12775 // only has to be registered.
12776 if (OMPBuilder->Config.isTargetDevice()) {
12777 // This could happen if the device compilation is invoked standalone.
12778 if (!hasTargetRegionEntryInfo(EntryInfo)) {
12779 return;
12780 }
12781 auto &Entry = OffloadEntriesTargetRegion[EntryInfo];
12782 Entry.setAddress(Addr);
12783 Entry.setID(ID);
12784 Entry.setFlags(Flags);
12785 } else {
12786 if (Flags == OffloadEntriesInfoManager::OMPTargetRegionEntryTargetRegion &&
12787 hasTargetRegionEntryInfo(EntryInfo, /*IgnoreAddressId*/ true))
12788 return;
12789 assert(!hasTargetRegionEntryInfo(EntryInfo) &&
12790 "Target region entry already registered!");
12791 OffloadEntryInfoTargetRegion Entry(OffloadingEntriesNum, Addr, ID, Flags);
12792 OffloadEntriesTargetRegion[EntryInfo] = Entry;
12793 ++OffloadingEntriesNum;
12794 }
12795 incrementTargetRegionEntryInfoCount(EntryInfo);
12796}
12797
12798bool OffloadEntriesInfoManager::hasTargetRegionEntryInfo(
12799 TargetRegionEntryInfo EntryInfo, bool IgnoreAddressId) const {
12800
12801 // Update the EntryInfo with the next available count for this location.
12802 EntryInfo.Count = getTargetRegionEntryInfoCount(EntryInfo);
12803
12804 auto It = OffloadEntriesTargetRegion.find(x: EntryInfo);
12805 if (It == OffloadEntriesTargetRegion.end()) {
12806 return false;
12807 }
12808 // Fail if this entry is already registered.
12809 if (!IgnoreAddressId && (It->second.getAddress() || It->second.getID()))
12810 return false;
12811 return true;
12812}
12813
12814void OffloadEntriesInfoManager::actOnTargetRegionEntriesInfo(
12815 const OffloadTargetRegionEntryInfoActTy &Action) {
12816 // Scan all target region entries and perform the provided action.
12817 for (const auto &It : OffloadEntriesTargetRegion) {
12818 Action(It.first, It.second);
12819 }
12820}
12821
12822void OffloadEntriesInfoManager::initializeDeviceGlobalVarEntryInfo(
12823 StringRef Name, OMPTargetGlobalVarEntryKind Flags, unsigned Order) {
12824 OffloadEntriesDeviceGlobalVar.try_emplace(Key: Name, Args&: Order, Args&: Flags);
12825 ++OffloadingEntriesNum;
12826}
12827
12828void OffloadEntriesInfoManager::registerDeviceGlobalVarEntryInfo(
12829 StringRef VarName, Constant *Addr, int64_t VarSize,
12830 OMPTargetGlobalVarEntryKind Flags, GlobalValue::LinkageTypes Linkage) {
12831 if (OMPBuilder->Config.isTargetDevice()) {
12832 // This could happen if the device compilation is invoked standalone.
12833 if (!hasDeviceGlobalVarEntryInfo(VarName))
12834 return;
12835 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
12836 if (Entry.getAddress() && hasDeviceGlobalVarEntryInfo(VarName)) {
12837 if (Entry.getVarSize() == 0) {
12838 Entry.setVarSize(VarSize);
12839 Entry.setLinkage(Linkage);
12840 }
12841 return;
12842 }
12843 Entry.setVarSize(VarSize);
12844 Entry.setLinkage(Linkage);
12845 Entry.setAddress(Addr);
12846 } else {
12847 if (hasDeviceGlobalVarEntryInfo(VarName)) {
12848 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
12849 assert(Entry.isValid() && Entry.getFlags() == Flags &&
12850 "Entry not initialized!");
12851 if (Entry.getVarSize() == 0) {
12852 Entry.setVarSize(VarSize);
12853 Entry.setLinkage(Linkage);
12854 }
12855 return;
12856 }
12857 if (Flags == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect ||
12858 Flags ==
12859 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirectVTable)
12860 OffloadEntriesDeviceGlobalVar.try_emplace(Key: VarName, Args&: OffloadingEntriesNum,
12861 Args&: Addr, Args&: VarSize, Args&: Flags, Args&: Linkage,
12862 Args: VarName.str());
12863 else
12864 OffloadEntriesDeviceGlobalVar.try_emplace(
12865 Key: VarName, Args&: OffloadingEntriesNum, Args&: Addr, Args&: VarSize, Args&: Flags, Args&: Linkage, Args: "");
12866 ++OffloadingEntriesNum;
12867 }
12868}
12869
12870void OffloadEntriesInfoManager::actOnDeviceGlobalVarEntriesInfo(
12871 const OffloadDeviceGlobalVarEntryInfoActTy &Action) {
12872 // Scan all target region entries and perform the provided action.
12873 for (const auto &E : OffloadEntriesDeviceGlobalVar)
12874 Action(E.getKey(), E.getValue());
12875}
12876
12877//===----------------------------------------------------------------------===//
12878// CanonicalLoopInfo
12879//===----------------------------------------------------------------------===//
12880
12881void CanonicalLoopInfo::collectControlBlocks(
12882 SmallVectorImpl<BasicBlock *> &BBs) {
12883 // We only count those BBs as control block for which we do not need to
12884 // reverse the CFG, i.e. not the loop body which can contain arbitrary control
12885 // flow. For consistency, this also means we do not add the Body block, which
12886 // is just the entry to the body code.
12887 BBs.reserve(N: BBs.size() + 6);
12888 BBs.append(IL: {getPreheader(), Header, Cond, Latch, Exit, getAfter()});
12889}
12890
12891BasicBlock *CanonicalLoopInfo::getPreheader() const {
12892 assert(isValid() && "Requires a valid canonical loop");
12893 for (BasicBlock *Pred : predecessors(BB: Header)) {
12894 if (Pred != Latch)
12895 return Pred;
12896 }
12897 llvm_unreachable("Missing preheader");
12898}
12899
12900void CanonicalLoopInfo::setTripCount(Value *TripCount) {
12901 assert(isValid() && "Requires a valid canonical loop");
12902
12903 Instruction *CmpI = &getCond()->front();
12904 assert(isa<CmpInst>(CmpI) && "First inst must compare IV with TripCount");
12905 CmpI->setOperand(i: 1, Val: TripCount);
12906
12907#ifndef NDEBUG
12908 assertOK();
12909#endif
12910}
12911
12912void CanonicalLoopInfo::mapIndVar(
12913 llvm::function_ref<Value *(Instruction *)> Updater) {
12914 assert(isValid() && "Requires a valid canonical loop");
12915
12916 Instruction *OldIV = getIndVar();
12917
12918 // Record all uses excluding those introduced by the updater. Uses by the
12919 // CanonicalLoopInfo itself to keep track of the number of iterations are
12920 // excluded.
12921 SmallVector<Use *> ReplacableUses;
12922 for (Use &U : OldIV->uses()) {
12923 auto *User = dyn_cast<Instruction>(Val: U.getUser());
12924 if (!User)
12925 continue;
12926 if (User->getParent() == getCond())
12927 continue;
12928 if (User->getParent() == getLatch())
12929 continue;
12930 ReplacableUses.push_back(Elt: &U);
12931 }
12932
12933 // Run the updater that may introduce new uses
12934 Value *NewIV = Updater(OldIV);
12935
12936 // Replace the old uses with the value returned by the updater.
12937 for (Use *U : ReplacableUses)
12938 U->set(NewIV);
12939
12940#ifndef NDEBUG
12941 assertOK();
12942#endif
12943}
12944
12945void CanonicalLoopInfo::assertOK() const {
12946#ifndef NDEBUG
12947 // No constraints if this object currently does not describe a loop.
12948 if (!isValid())
12949 return;
12950
12951 BasicBlock *Preheader = getPreheader();
12952 BasicBlock *Body = getBody();
12953 BasicBlock *After = getAfter();
12954
12955 // Verify standard control-flow we use for OpenMP loops.
12956 assert(Preheader);
12957 assert(isa<UncondBrInst>(Preheader->getTerminator()) &&
12958 "Preheader must terminate with unconditional branch");
12959 assert(Preheader->getSingleSuccessor() == Header &&
12960 "Preheader must jump to header");
12961
12962 assert(Header);
12963 assert(isa<UncondBrInst>(Header->getTerminator()) &&
12964 "Header must terminate with unconditional branch");
12965 assert(Header->getSingleSuccessor() == Cond &&
12966 "Header must jump to exiting block");
12967
12968 assert(Cond);
12969 assert(Cond->getSinglePredecessor() == Header &&
12970 "Exiting block only reachable from header");
12971
12972 assert(isa<CondBrInst>(Cond->getTerminator()) &&
12973 "Exiting block must terminate with conditional branch");
12974 assert(cast<CondBrInst>(Cond->getTerminator())->getSuccessor(0) == Body &&
12975 "Exiting block's first successor jump to the body");
12976 assert(cast<CondBrInst>(Cond->getTerminator())->getSuccessor(1) == Exit &&
12977 "Exiting block's second successor must exit the loop");
12978
12979 assert(Body);
12980 assert(Body->getSinglePredecessor() == Cond &&
12981 "Body only reachable from exiting block");
12982 assert(!isa<PHINode>(Body->front()));
12983
12984 assert(Latch);
12985 assert(isa<UncondBrInst>(Latch->getTerminator()) &&
12986 "Latch must terminate with unconditional branch");
12987 assert(Latch->getSingleSuccessor() == Header && "Latch must jump to header");
12988 // TODO: To support simple redirecting of the end of the body code that has
12989 // multiple; introduce another auxiliary basic block like preheader and after.
12990 assert(Latch->getSinglePredecessor() != nullptr);
12991 assert(!isa<PHINode>(Latch->front()));
12992
12993 assert(Exit);
12994 assert(isa<UncondBrInst>(Exit->getTerminator()) &&
12995 "Exit block must terminate with unconditional branch");
12996 assert(Exit->getSingleSuccessor() == After &&
12997 "Exit block must jump to after block");
12998
12999 assert(After);
13000 assert(After->getSinglePredecessor() == Exit &&
13001 "After block only reachable from exit block");
13002 assert(After->empty() || !isa<PHINode>(After->front()));
13003
13004 Instruction *IndVar = getIndVar();
13005 assert(IndVar && "Canonical induction variable not found?");
13006 assert(isa<IntegerType>(IndVar->getType()) &&
13007 "Induction variable must be an integer");
13008 assert(cast<PHINode>(IndVar)->getParent() == Header &&
13009 "Induction variable must be a PHI in the loop header");
13010 assert(cast<PHINode>(IndVar)->getIncomingBlock(0) == Preheader);
13011 assert(
13012 cast<ConstantInt>(cast<PHINode>(IndVar)->getIncomingValue(0))->isZero());
13013 assert(cast<PHINode>(IndVar)->getIncomingBlock(1) == Latch);
13014
13015 auto *NextIndVar = cast<PHINode>(IndVar)->getIncomingValue(1);
13016 assert(cast<Instruction>(NextIndVar)->getParent() == Latch);
13017 assert(cast<BinaryOperator>(NextIndVar)->getOpcode() == BinaryOperator::Add);
13018 assert(cast<BinaryOperator>(NextIndVar)->getOperand(0) == IndVar);
13019 assert(cast<ConstantInt>(cast<BinaryOperator>(NextIndVar)->getOperand(1))
13020 ->isOne());
13021
13022 Value *TripCount = getTripCount();
13023 assert(TripCount && "Loop trip count not found?");
13024 assert(IndVar->getType() == TripCount->getType() &&
13025 "Trip count and induction variable must have the same type");
13026
13027 auto *CmpI = cast<CmpInst>(&Cond->front());
13028 assert(CmpI->getPredicate() == CmpInst::ICMP_ULT &&
13029 "Exit condition must be a signed less-than comparison");
13030 assert(CmpI->getOperand(0) == IndVar &&
13031 "Exit condition must compare the induction variable");
13032 assert(CmpI->getOperand(1) == TripCount &&
13033 "Exit condition must compare with the trip count");
13034#endif
13035}
13036
13037void CanonicalLoopInfo::invalidate() {
13038 Header = nullptr;
13039 Cond = nullptr;
13040 Latch = nullptr;
13041 Exit = nullptr;
13042}
13043