1//===- OpenMPIRBuilder.cpp - Builder for LLVM-IR for OpenMP directives ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9///
10/// This file implements the OpenMPIRBuilder class, which is used as a
11/// convenient way to create LLVM instructions for OpenMP directives.
12///
13//===----------------------------------------------------------------------===//
14
15#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
16#include "llvm/ADT/SmallBitVector.h"
17#include "llvm/ADT/SmallSet.h"
18#include "llvm/ADT/SmallVectorExtras.h"
19#include "llvm/ADT/StringExtras.h"
20#include "llvm/ADT/StringRef.h"
21#include "llvm/Analysis/AssumptionCache.h"
22#include "llvm/Analysis/LoopInfo.h"
23#include "llvm/Analysis/PostDominators.h"
24#include "llvm/Analysis/TargetLibraryInfo.h"
25#include "llvm/Bitcode/BitcodeReader.h"
26#include "llvm/Frontend/Offloading/Utility.h"
27#include "llvm/Frontend/OpenMP/OMPGridValues.h"
28#include "llvm/IR/Attributes.h"
29#include "llvm/IR/BasicBlock.h"
30#include "llvm/IR/CFG.h"
31#include "llvm/IR/CallingConv.h"
32#include "llvm/IR/Constant.h"
33#include "llvm/IR/Constants.h"
34#include "llvm/IR/DIBuilder.h"
35#include "llvm/IR/DebugInfoMetadata.h"
36#include "llvm/IR/DerivedTypes.h"
37#include "llvm/IR/Function.h"
38#include "llvm/IR/GlobalVariable.h"
39#include "llvm/IR/IRBuilder.h"
40#include "llvm/IR/InstIterator.h"
41#include "llvm/IR/LLVMContext.h"
42#include "llvm/IR/MDBuilder.h"
43#include "llvm/IR/Metadata.h"
44#include "llvm/IR/PassInstrumentation.h"
45#include "llvm/IR/PassManager.h"
46#include "llvm/IR/ReplaceConstant.h"
47#include "llvm/IR/Value.h"
48#include "llvm/Support/CommandLine.h"
49#include "llvm/Support/Error.h"
50#include "llvm/Support/ErrorHandling.h"
51#include "llvm/Support/NVVMAttributes.h"
52#include "llvm/Support/VirtualFileSystem.h"
53#include "llvm/Transforms/Utils/BasicBlockUtils.h"
54#include "llvm/Transforms/Utils/Cloning.h"
55#include "llvm/Transforms/Utils/CodeExtractor.h"
56
57#include <cstdint>
58#include <optional>
59
60#define DEBUG_TYPE "openmp-ir-builder"
61
62using namespace llvm;
63using namespace omp;
64
65static cl::opt<bool>
66 OptimisticAttributes("openmp-ir-builder-optimistic-attributes", cl::Hidden,
67 cl::desc("Use optimistic attributes describing "
68 "'as-if' properties of runtime calls."),
69 cl::init(Val: false));
70
71static cl::opt<bool> UseDefaultMaxThreads(
72 "openmp-ir-builder-use-default-max-threads", cl::Hidden,
73 cl::desc("Use a default max threads if none is provided."), cl::init(Val: true));
74
75#ifndef NDEBUG
76/// Return whether IP1 and IP2 are ambiguous, i.e. that inserting instructions
77/// at position IP1 may change the meaning of IP2 or vice-versa. This is because
78/// an InsertPoint stores the instruction before something is inserted. For
79/// instance, if both point to the same instruction, two IRBuilders alternating
80/// creating instruction will cause the instructions to be interleaved.
81static bool isConflictIP(IRBuilder<>::InsertPoint IP1,
82 IRBuilder<>::InsertPoint IP2) {
83 if (!IP1.isValid() || !IP2.isValid())
84 return false;
85 return IP1 == IP2;
86}
87
88static bool isValidWorkshareLoopScheduleType(OMPScheduleType SchedType) {
89 // Valid ordered/unordered and base algorithm combinations.
90 switch (SchedType & ~OMPScheduleType::MonotonicityMask) {
91 case OMPScheduleType::UnorderedStaticChunked:
92 case OMPScheduleType::UnorderedStatic:
93 case OMPScheduleType::UnorderedDynamicChunked:
94 case OMPScheduleType::UnorderedGuidedChunked:
95 case OMPScheduleType::UnorderedRuntime:
96 case OMPScheduleType::UnorderedAuto:
97 case OMPScheduleType::UnorderedTrapezoidal:
98 case OMPScheduleType::UnorderedGreedy:
99 case OMPScheduleType::UnorderedBalanced:
100 case OMPScheduleType::UnorderedGuidedIterativeChunked:
101 case OMPScheduleType::UnorderedGuidedAnalyticalChunked:
102 case OMPScheduleType::UnorderedSteal:
103 case OMPScheduleType::UnorderedStaticBalancedChunked:
104 case OMPScheduleType::UnorderedGuidedSimd:
105 case OMPScheduleType::UnorderedRuntimeSimd:
106 case OMPScheduleType::OrderedStaticChunked:
107 case OMPScheduleType::OrderedStatic:
108 case OMPScheduleType::OrderedDynamicChunked:
109 case OMPScheduleType::OrderedGuidedChunked:
110 case OMPScheduleType::OrderedRuntime:
111 case OMPScheduleType::OrderedAuto:
112 case OMPScheduleType::OrderdTrapezoidal:
113 case OMPScheduleType::NomergeUnorderedStaticChunked:
114 case OMPScheduleType::NomergeUnorderedStatic:
115 case OMPScheduleType::NomergeUnorderedDynamicChunked:
116 case OMPScheduleType::NomergeUnorderedGuidedChunked:
117 case OMPScheduleType::NomergeUnorderedRuntime:
118 case OMPScheduleType::NomergeUnorderedAuto:
119 case OMPScheduleType::NomergeUnorderedTrapezoidal:
120 case OMPScheduleType::NomergeUnorderedGreedy:
121 case OMPScheduleType::NomergeUnorderedBalanced:
122 case OMPScheduleType::NomergeUnorderedGuidedIterativeChunked:
123 case OMPScheduleType::NomergeUnorderedGuidedAnalyticalChunked:
124 case OMPScheduleType::NomergeUnorderedSteal:
125 case OMPScheduleType::NomergeOrderedStaticChunked:
126 case OMPScheduleType::NomergeOrderedStatic:
127 case OMPScheduleType::NomergeOrderedDynamicChunked:
128 case OMPScheduleType::NomergeOrderedGuidedChunked:
129 case OMPScheduleType::NomergeOrderedRuntime:
130 case OMPScheduleType::NomergeOrderedAuto:
131 case OMPScheduleType::NomergeOrderedTrapezoidal:
132 case OMPScheduleType::OrderedDistributeChunked:
133 case OMPScheduleType::OrderedDistribute:
134 break;
135 default:
136 return false;
137 }
138
139 // Must not set both monotonicity modifiers at the same time.
140 OMPScheduleType MonotonicityFlags =
141 SchedType & OMPScheduleType::MonotonicityMask;
142 if (MonotonicityFlags == OMPScheduleType::MonotonicityMask)
143 return false;
144
145 return true;
146}
147#endif
148
149/// This is a wrapper over IRBuilderBase::restoreIP that also restores a current
150/// debug location when the insert point is at the end of a block. It picks a
151/// location scoped to the current function: the block's last instruction
152/// location if the block is non-empty, otherwise a location synthesized from
153/// the function's subprogram (when the function has debug info).
154static void restoreIPandDebugLoc(llvm::IRBuilderBase &Builder,
155 llvm::IRBuilderBase::InsertPoint IP) {
156 Builder.restoreIP(IP);
157 // When IP points at a real instruction, restoreIP (SetInsertPoint) already
158 // set the debug location from that instruction, so leave it alone.
159 llvm::BasicBlock *BB = Builder.GetInsertBlock();
160 if (Builder.GetInsertPoint() != BB->end())
161 return;
162
163 // At the end of a block, pick a location guaranteed to belong to the current
164 // insertion function's subprogram. Prefer the block's own last instruction;
165 // otherwise synthesize a location from the function's subprogram.
166 if (!BB->empty())
167 Builder.SetCurrentDebugLocation(BB->back().getDebugLoc());
168 else if (llvm::DISubprogram *FSP =
169 BB->getParent() ? BB->getParent()->getSubprogram() : nullptr) {
170 unsigned Line = FSP->getScopeLine() ? FSP->getScopeLine() : FSP->getLine();
171 Builder.SetCurrentDebugLocation(
172 llvm::DILocation::get(Context&: FSP->getContext(), Line, /*Column=*/0, Scope: FSP));
173 }
174}
175
176static bool hasGridValue(const Triple &T) {
177 return T.isAMDGPU() || T.isNVPTX() || T.isSPIRV();
178}
179
180static const omp::GV &getGridValue(const Triple &T, Function *Kernel) {
181 if (T.isAMDGPU()) {
182 StringRef Features =
183 Kernel->getFnAttribute(Kind: "target-features").getValueAsString();
184 if (Features.count(Str: "+wavefrontsize64"))
185 return omp::getAMDGPUGridValues<64>();
186 return omp::getAMDGPUGridValues<32>();
187 }
188 if (T.isNVPTX())
189 return omp::NVPTXGridValues;
190 if (T.isSPIRV())
191 return omp::SPIRVGridValues;
192 llvm_unreachable("No grid value available for this architecture!");
193}
194
195/// Determine which scheduling algorithm to use, determined from schedule clause
196/// arguments.
197static OMPScheduleType
198getOpenMPBaseScheduleType(llvm::omp::ScheduleKind ClauseKind, bool HasChunks,
199 bool HasSimdModifier, bool HasDistScheduleChunks) {
200 // Currently, the default schedule it static.
201 switch (ClauseKind) {
202 case OMP_SCHEDULE_Default:
203 case OMP_SCHEDULE_Static:
204 return HasChunks ? OMPScheduleType::BaseStaticChunked
205 : OMPScheduleType::BaseStatic;
206 case OMP_SCHEDULE_Dynamic:
207 return OMPScheduleType::BaseDynamicChunked;
208 case OMP_SCHEDULE_Guided:
209 return HasSimdModifier ? OMPScheduleType::BaseGuidedSimd
210 : OMPScheduleType::BaseGuidedChunked;
211 case OMP_SCHEDULE_Auto:
212 return llvm::omp::OMPScheduleType::BaseAuto;
213 case OMP_SCHEDULE_Runtime:
214 return HasSimdModifier ? OMPScheduleType::BaseRuntimeSimd
215 : OMPScheduleType::BaseRuntime;
216 case OMP_SCHEDULE_Distribute:
217 return HasDistScheduleChunks ? OMPScheduleType::BaseDistributeChunked
218 : OMPScheduleType::BaseDistribute;
219 }
220 llvm_unreachable("unhandled schedule clause argument");
221}
222
223/// Adds ordering modifier flags to schedule type.
224static OMPScheduleType
225getOpenMPOrderingScheduleType(OMPScheduleType BaseScheduleType,
226 bool HasOrderedClause) {
227 assert((BaseScheduleType & OMPScheduleType::ModifierMask) ==
228 OMPScheduleType::None &&
229 "Must not have ordering nor monotonicity flags already set");
230
231 OMPScheduleType OrderingModifier = HasOrderedClause
232 ? OMPScheduleType::ModifierOrdered
233 : OMPScheduleType::ModifierUnordered;
234 OMPScheduleType OrderingScheduleType = BaseScheduleType | OrderingModifier;
235
236 // Unsupported combinations
237 if (OrderingScheduleType ==
238 (OMPScheduleType::BaseGuidedSimd | OMPScheduleType::ModifierOrdered))
239 return OMPScheduleType::OrderedGuidedChunked;
240 else if (OrderingScheduleType == (OMPScheduleType::BaseRuntimeSimd |
241 OMPScheduleType::ModifierOrdered))
242 return OMPScheduleType::OrderedRuntime;
243
244 return OrderingScheduleType;
245}
246
247/// Adds monotonicity modifier flags to schedule type.
248static OMPScheduleType
249getOpenMPMonotonicityScheduleType(OMPScheduleType ScheduleType,
250 bool HasSimdModifier, bool HasMonotonic,
251 bool HasNonmonotonic, bool HasOrderedClause) {
252 assert((ScheduleType & OMPScheduleType::MonotonicityMask) ==
253 OMPScheduleType::None &&
254 "Must not have monotonicity flags already set");
255 assert((!HasMonotonic || !HasNonmonotonic) &&
256 "Monotonic and Nonmonotonic are contradicting each other");
257
258 if (HasMonotonic) {
259 return ScheduleType | OMPScheduleType::ModifierMonotonic;
260 } else if (HasNonmonotonic) {
261 return ScheduleType | OMPScheduleType::ModifierNonmonotonic;
262 } else {
263 // OpenMP 5.1, 2.11.4 Worksharing-Loop Construct, Description.
264 // If the static schedule kind is specified or if the ordered clause is
265 // specified, and if the nonmonotonic modifier is not specified, the
266 // effect is as if the monotonic modifier is specified. Otherwise, unless
267 // the monotonic modifier is specified, the effect is as if the
268 // nonmonotonic modifier is specified.
269 OMPScheduleType BaseScheduleType =
270 ScheduleType & ~OMPScheduleType::ModifierMask;
271 if ((BaseScheduleType == OMPScheduleType::BaseStatic) ||
272 (BaseScheduleType == OMPScheduleType::BaseStaticChunked) ||
273 HasOrderedClause) {
274 // The monotonic is used by default in openmp runtime library, so no need
275 // to set it.
276 return ScheduleType;
277 } else {
278 return ScheduleType | OMPScheduleType::ModifierNonmonotonic;
279 }
280 }
281}
282
283/// Determine the schedule type using schedule and ordering clause arguments.
284static OMPScheduleType
285computeOpenMPScheduleType(ScheduleKind ClauseKind, bool HasChunks,
286 bool HasSimdModifier, bool HasMonotonicModifier,
287 bool HasNonmonotonicModifier, bool HasOrderedClause,
288 bool HasDistScheduleChunks) {
289 OMPScheduleType BaseSchedule = getOpenMPBaseScheduleType(
290 ClauseKind, HasChunks, HasSimdModifier, HasDistScheduleChunks);
291 OMPScheduleType OrderedSchedule =
292 getOpenMPOrderingScheduleType(BaseScheduleType: BaseSchedule, HasOrderedClause);
293 OMPScheduleType Result = getOpenMPMonotonicityScheduleType(
294 ScheduleType: OrderedSchedule, HasSimdModifier, HasMonotonic: HasMonotonicModifier,
295 HasNonmonotonic: HasNonmonotonicModifier, HasOrderedClause);
296
297 assert(isValidWorkshareLoopScheduleType(Result));
298 return Result;
299}
300
301/// Given a function, if it represents the entry point of a target kernel, this
302/// returns the execution mode flags associated with that kernel.
303static std::optional<omp::OMPTgtExecModeFlags>
304getTargetKernelExecMode(Function &Kernel) {
305 CallInst *TargetInitCall = nullptr;
306 for (Instruction &Inst : Kernel.getEntryBlock()) {
307 if (auto *Call = dyn_cast<CallInst>(Val: &Inst)) {
308 if (Call->getCalledFunction()->getName() == "__kmpc_target_init") {
309 TargetInitCall = Call;
310 break;
311 }
312 }
313 }
314
315 if (!TargetInitCall)
316 return std::nullopt;
317
318 // Get the kernel mode information from the global variable associated to the
319 // first argument to the call to __kmpc_target_init. Refer to
320 // createTargetInit() to see how this is initialized.
321 Value *InitOperand = TargetInitCall->getArgOperand(i: 0);
322 GlobalVariable *KernelEnv = nullptr;
323 if (auto *Cast = dyn_cast<ConstantExpr>(Val: InitOperand))
324 KernelEnv = cast<GlobalVariable>(Val: Cast->getOperand(i_nocapture: 0));
325 else
326 KernelEnv = cast<GlobalVariable>(Val: InitOperand);
327 auto *KernelEnvInit = cast<ConstantStruct>(Val: KernelEnv->getInitializer());
328 auto *ConfigEnv = cast<ConstantStruct>(Val: KernelEnvInit->getOperand(i_nocapture: 0));
329 auto *KernelMode = cast<ConstantInt>(Val: ConfigEnv->getOperand(i_nocapture: 2));
330 return static_cast<OMPTgtExecModeFlags>(KernelMode->getZExtValue());
331}
332
333static bool isGenericKernel(Function &Fn) {
334 std::optional<omp::OMPTgtExecModeFlags> ExecMode =
335 getTargetKernelExecMode(Kernel&: Fn);
336 return !ExecMode || (*ExecMode & OMP_TGT_EXEC_MODE_GENERIC);
337}
338
339/// Make \p Source branch to \p Target.
340///
341/// Handles two situations:
342/// * \p Source already has an unconditional branch.
343/// * \p Source is a degenerate block (no terminator because the BB is
344/// the current head of the IR construction).
345static void redirectTo(BasicBlock *Source, BasicBlock *Target, DebugLoc DL) {
346 if (Instruction *Term = Source->getTerminatorOrNull()) {
347 auto *Br = cast<UncondBrInst>(Val: Term);
348 BasicBlock *Succ = Br->getSuccessor();
349 Succ->removePredecessor(Pred: Source, /*KeepOneInputPHIs=*/true);
350 Br->setSuccessor(Target);
351 return;
352 }
353
354 auto *NewBr = UncondBrInst::Create(Target, InsertBefore: Source);
355 NewBr->setDebugLoc(DL);
356}
357
358void llvm::spliceBB(IRBuilderBase::InsertPoint IP, BasicBlock *New,
359 bool CreateBranch, DebugLoc DL) {
360 assert(New->getFirstInsertionPt() == New->begin() &&
361 "Target BB must not have PHI nodes");
362
363 // Move instructions to new block.
364 BasicBlock *Old = IP.getNodeParent();
365 // If the `Old` block is empty then there are no instructions to move. But in
366 // the new debug scheme, it could have trailing debug records which will be
367 // moved to `New` in `spliceDebugInfoEmptyBlock`. We dont want that for 2
368 // reasons:
369 // 1. If `New` is also empty, `BasicBlock::splice` crashes.
370 // 2. Even if `New` is not empty, the rationale to move those records to `New`
371 // (in `spliceDebugInfoEmptyBlock`) does not apply here. That function
372 // assumes that `Old` is optimized out and is going away. This is not the case
373 // here. The `Old` block is still being used e.g. a branch instruction is
374 // added to it later in this function.
375 // So we call `BasicBlock::splice` only when `Old` is not empty.
376 if (!Old->empty())
377 New->splice(ToIt: New->begin(), FromBB: Old, FromBeginIt: IP, FromEndIt: Old->end());
378
379 if (CreateBranch) {
380 auto *NewBr = UncondBrInst::Create(Target: New, InsertBefore: Old);
381 NewBr->setDebugLoc(DL);
382 }
383}
384
385void llvm::spliceBB(IRBuilder<> &Builder, BasicBlock *New, bool CreateBranch) {
386 DebugLoc DebugLoc = Builder.getCurrentDebugLocation();
387 BasicBlock *Old = Builder.GetInsertBlock();
388
389 spliceBB(IP: Builder.saveIP(), New, CreateBranch, DL: DebugLoc);
390 if (CreateBranch)
391 Builder.SetInsertPoint(Old->getTerminator());
392 else
393 Builder.SetInsertPoint(Old);
394
395 // SetInsertPoint also updates the Builder's debug location, but we want to
396 // keep the one the Builder was configured to use.
397 Builder.SetCurrentDebugLocation(DebugLoc);
398}
399
400BasicBlock *llvm::splitBB(IRBuilderBase::InsertPoint IP, bool CreateBranch,
401 DebugLoc DL, llvm::Twine Name) {
402 BasicBlock *Old = IP.getNodeParent();
403 BasicBlock *New = BasicBlock::Create(
404 Context&: Old->getContext(), Name: Name.isTriviallyEmpty() ? Old->getName() : Name,
405 Parent: Old->getParent(), InsertBefore: Old->getNextNode());
406 spliceBB(IP, New, CreateBranch, DL);
407 New->replaceSuccessorsPhiUsesWith(Old, New);
408 return New;
409}
410
411BasicBlock *llvm::splitBB(IRBuilderBase &Builder, bool CreateBranch,
412 llvm::Twine Name) {
413 DebugLoc DebugLoc = Builder.getCurrentDebugLocation();
414 BasicBlock *Old = Builder.GetInsertBlock();
415 BasicBlock *New = splitBB(IP: Builder.saveIP(), CreateBranch, DL: DebugLoc, Name);
416 if (CreateBranch)
417 Builder.SetInsertPoint(Old->getTerminator());
418 else
419 Builder.SetInsertPoint(Old);
420 // SetInsertPoint also updates the Builder's debug location, but we want to
421 // keep the one the Builder was configured to use.
422 Builder.SetCurrentDebugLocation(DebugLoc);
423 return New;
424}
425
426BasicBlock *llvm::splitBB(IRBuilder<> &Builder, bool CreateBranch,
427 llvm::Twine Name) {
428 DebugLoc DebugLoc = Builder.getCurrentDebugLocation();
429 BasicBlock *Old = Builder.GetInsertBlock();
430 BasicBlock *New = splitBB(IP: Builder.saveIP(), CreateBranch, DL: DebugLoc, Name);
431 if (CreateBranch)
432 Builder.SetInsertPoint(Old->getTerminator());
433 else
434 Builder.SetInsertPoint(Old);
435 // SetInsertPoint also updates the Builder's debug location, but we want to
436 // keep the one the Builder was configured to use.
437 Builder.SetCurrentDebugLocation(DebugLoc);
438 return New;
439}
440
441BasicBlock *llvm::splitBBWithSuffix(IRBuilderBase &Builder, bool CreateBranch,
442 llvm::Twine Suffix) {
443 BasicBlock *Old = Builder.GetInsertBlock();
444 return splitBB(Builder, CreateBranch, Name: Old->getName() + Suffix);
445}
446
447// This function creates a fake integer value and a fake use for the integer
448// value. It returns the fake value created. This is useful in modeling the
449// extra arguments to the outlined functions.
450Value *createFakeIntVal(IRBuilderBase &Builder,
451 OpenMPIRBuilder::InsertPointTy OuterAllocaIP,
452 llvm::SmallVectorImpl<Instruction *> &ToBeDeleted,
453 OpenMPIRBuilder::InsertPointTy InnerAllocaIP,
454 const Twine &Name = "", bool AsPtr = true,
455 bool Is64Bit = false) {
456 Builder.restoreIP(IP: OuterAllocaIP);
457 IntegerType *IntTy = Is64Bit ? Builder.getInt64Ty() : Builder.getInt32Ty();
458 Instruction *FakeVal;
459 AllocaInst *FakeValAddr =
460 Builder.CreateAlloca(Ty: IntTy, ArraySize: nullptr, Name: Name + ".addr");
461 ToBeDeleted.push_back(Elt: FakeValAddr);
462
463 if (AsPtr) {
464 FakeVal = FakeValAddr;
465 // The runtime passes these extra arguments to the outlined function as
466 // generic pointers, so cast away a non-zero alloca address space.
467 if (FakeValAddr->getAddressSpace() != 0) {
468 FakeVal = cast<Instruction>(Val: Builder.CreateAddrSpaceCast(
469 V: FakeValAddr, DestTy: Builder.getPtrTy(), Name: Name + ".ascast"));
470 ToBeDeleted.push_back(Elt: FakeVal);
471 }
472 } else {
473 FakeVal = Builder.CreateLoad(Ty: IntTy, Ptr: FakeValAddr, Name: Name + ".val");
474 ToBeDeleted.push_back(Elt: FakeVal);
475 }
476
477 // Generate a fake use of this value
478 Builder.restoreIP(IP: InnerAllocaIP);
479 Instruction *UseFakeVal;
480 if (AsPtr) {
481 UseFakeVal = Builder.CreateLoad(Ty: IntTy, Ptr: FakeVal, Name: Name + ".use");
482 } else {
483 UseFakeVal = cast<BinaryOperator>(Val: Builder.CreateAdd(
484 LHS: FakeVal, RHS: Is64Bit ? Builder.getInt64(C: 10) : Builder.getInt32(C: 10)));
485 }
486 ToBeDeleted.push_back(Elt: UseFakeVal);
487 return FakeVal;
488}
489
490//===----------------------------------------------------------------------===//
491// OpenMPIRBuilderConfig
492//===----------------------------------------------------------------------===//
493
494namespace {
495LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();
496/// Values for bit flags for marking which requires clauses have been used.
497enum OpenMPOffloadingRequiresDirFlags {
498 /// flag undefined.
499 OMP_REQ_UNDEFINED = 0x000,
500 /// no requires directive present.
501 OMP_REQ_NONE = 0x001,
502 /// reverse_offload clause.
503 OMP_REQ_REVERSE_OFFLOAD = 0x002,
504 /// unified_address clause.
505 OMP_REQ_UNIFIED_ADDRESS = 0x004,
506 /// unified_shared_memory clause.
507 OMP_REQ_UNIFIED_SHARED_MEMORY = 0x008,
508 /// dynamic_allocators clause.
509 OMP_REQ_DYNAMIC_ALLOCATORS = 0x010,
510 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_REQ_DYNAMIC_ALLOCATORS)
511};
512
513class OMPCodeExtractor : public CodeExtractor {
514public:
515 OMPCodeExtractor(OpenMPIRBuilder &OMPBuilder, ArrayRef<BasicBlock *> BBs,
516 DominatorTree *DT = nullptr, bool AggregateArgs = false,
517 BlockFrequencyInfo *BFI = nullptr,
518 BranchProbabilityInfo *BPI = nullptr,
519 AssumptionCache *AC = nullptr, bool AllowVarArgs = false,
520 bool AllowAlloca = false,
521 BasicBlock *AllocationBlock = nullptr,
522 ArrayRef<BasicBlock *> DeallocationBlocks = {},
523 std::string Suffix = "", bool ArgsInZeroAddressSpace = false)
524 : CodeExtractor(BBs, DT, AggregateArgs, BFI, BPI, AC, AllowVarArgs,
525 AllowAlloca, AllocationBlock, DeallocationBlocks, Suffix,
526 ArgsInZeroAddressSpace),
527 OMPBuilder(OMPBuilder) {}
528
529 virtual ~OMPCodeExtractor() = default;
530
531protected:
532 OpenMPIRBuilder &OMPBuilder;
533};
534
535class DeviceSharedMemCodeExtractor : public OMPCodeExtractor {
536public:
537 using OMPCodeExtractor::OMPCodeExtractor;
538 virtual ~DeviceSharedMemCodeExtractor() = default;
539
540protected:
541 virtual Instruction *
542 allocateVar(IRBuilder<>::InsertPoint AllocaIP, DebugLoc DL, Type *VarType,
543 const Twine &Name = Twine(""),
544 AddrSpaceCastInst **CastedAlloc = nullptr) override {
545 return OMPBuilder.createOMPAllocShared(Loc: {AllocaIP, DL}, VarType, Name);
546 }
547
548 virtual Instruction *deallocateVar(IRBuilder<>::InsertPoint DeallocIP,
549 DebugLoc DL, Value *Var,
550 Type *VarType) override {
551 return OMPBuilder.createOMPFreeShared(Loc: {DeallocIP, DL}, Addr: Var, VarType);
552 }
553};
554
555/// Helper storing information about regions to outline using device shared
556/// memory for intermediate allocations.
557struct DeviceSharedMemOutlineInfo : public OpenMPIRBuilder::OutlineInfo {
558 OpenMPIRBuilder &OMPBuilder;
559
560 DeviceSharedMemOutlineInfo(OpenMPIRBuilder &OMPBuilder)
561 : OMPBuilder(OMPBuilder) {}
562 virtual ~DeviceSharedMemOutlineInfo() = default;
563
564 virtual std::unique_ptr<CodeExtractor>
565 createCodeExtractor(ArrayRef<BasicBlock *> Blocks,
566 bool ArgsInZeroAddressSpace,
567 Twine Suffix = Twine("")) override;
568};
569
570} // anonymous namespace
571
572OpenMPIRBuilderConfig::OpenMPIRBuilderConfig()
573 : RequiresFlags(OMP_REQ_UNDEFINED) {}
574
575OpenMPIRBuilderConfig::OpenMPIRBuilderConfig(
576 bool IsTargetDevice, bool IsGPU, bool OpenMPOffloadMandatory,
577 bool HasRequiresReverseOffload, bool HasRequiresUnifiedAddress,
578 bool HasRequiresUnifiedSharedMemory, bool HasRequiresDynamicAllocators)
579 : IsTargetDevice(IsTargetDevice), IsGPU(IsGPU),
580 OpenMPOffloadMandatory(OpenMPOffloadMandatory),
581 RequiresFlags(OMP_REQ_UNDEFINED) {
582 if (HasRequiresReverseOffload)
583 RequiresFlags |= OMP_REQ_REVERSE_OFFLOAD;
584 if (HasRequiresUnifiedAddress)
585 RequiresFlags |= OMP_REQ_UNIFIED_ADDRESS;
586 if (HasRequiresUnifiedSharedMemory)
587 RequiresFlags |= OMP_REQ_UNIFIED_SHARED_MEMORY;
588 if (HasRequiresDynamicAllocators)
589 RequiresFlags |= OMP_REQ_DYNAMIC_ALLOCATORS;
590}
591
592bool OpenMPIRBuilderConfig::hasRequiresReverseOffload() const {
593 return RequiresFlags & OMP_REQ_REVERSE_OFFLOAD;
594}
595
596bool OpenMPIRBuilderConfig::hasRequiresUnifiedAddress() const {
597 return RequiresFlags & OMP_REQ_UNIFIED_ADDRESS;
598}
599
600bool OpenMPIRBuilderConfig::hasRequiresUnifiedSharedMemory() const {
601 return RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY;
602}
603
604bool OpenMPIRBuilderConfig::hasRequiresDynamicAllocators() const {
605 return RequiresFlags & OMP_REQ_DYNAMIC_ALLOCATORS;
606}
607
608int64_t OpenMPIRBuilderConfig::getRequiresFlags() const {
609 return hasRequiresFlags() ? RequiresFlags
610 : static_cast<int64_t>(OMP_REQ_NONE);
611}
612
613void OpenMPIRBuilderConfig::setHasRequiresReverseOffload(bool Value) {
614 if (Value)
615 RequiresFlags |= OMP_REQ_REVERSE_OFFLOAD;
616 else
617 RequiresFlags &= ~OMP_REQ_REVERSE_OFFLOAD;
618}
619
620void OpenMPIRBuilderConfig::setHasRequiresUnifiedAddress(bool Value) {
621 if (Value)
622 RequiresFlags |= OMP_REQ_UNIFIED_ADDRESS;
623 else
624 RequiresFlags &= ~OMP_REQ_UNIFIED_ADDRESS;
625}
626
627void OpenMPIRBuilderConfig::setHasRequiresUnifiedSharedMemory(bool Value) {
628 if (Value)
629 RequiresFlags |= OMP_REQ_UNIFIED_SHARED_MEMORY;
630 else
631 RequiresFlags &= ~OMP_REQ_UNIFIED_SHARED_MEMORY;
632}
633
634void OpenMPIRBuilderConfig::setHasRequiresDynamicAllocators(bool Value) {
635 if (Value)
636 RequiresFlags |= OMP_REQ_DYNAMIC_ALLOCATORS;
637 else
638 RequiresFlags &= ~OMP_REQ_DYNAMIC_ALLOCATORS;
639}
640
641//===----------------------------------------------------------------------===//
642// OpenMPIRBuilder
643//===----------------------------------------------------------------------===//
644
645void OpenMPIRBuilder::getKernelArgsVector(TargetKernelArgs &KernelArgs,
646 IRBuilderBase &Builder,
647 SmallVector<Value *> &ArgsVector) {
648 Value *Version = Builder.getInt32(OMP_KERNEL_ARG_VERSION);
649 Value *PointerNum = Builder.getInt32(C: KernelArgs.NumTargetItems);
650 auto Int32Ty = Type::getInt32Ty(C&: Builder.getContext());
651 constexpr size_t MaxDim = 3;
652 Value *ZeroArray = Constant::getNullValue(Ty: ArrayType::get(ElementType: Int32Ty, NumElements: MaxDim));
653
654 Value *HasNoWaitFlag = Builder.getInt64(C: KernelArgs.HasNoWait);
655
656 Value *DynCGroupMemFallbackFlag =
657 Builder.getInt64(C: static_cast<uint64_t>(KernelArgs.DynCGroupMemFallback));
658 DynCGroupMemFallbackFlag = Builder.CreateShl(LHS: DynCGroupMemFallbackFlag, RHS: 2);
659
660 Value *StrictBlocksFlag = Builder.getInt64(C: KernelArgs.StrictBlocks);
661 Value *StrictThreadsFlag = Builder.getInt64(C: KernelArgs.StrictThreads);
662
663 StrictBlocksFlag = Builder.CreateShl(LHS: StrictBlocksFlag, RHS: 6);
664 StrictThreadsFlag = Builder.CreateShl(LHS: StrictThreadsFlag, RHS: 7);
665
666 Value *Flags = Builder.CreateOr(LHS: HasNoWaitFlag, RHS: DynCGroupMemFallbackFlag);
667 Flags = Builder.CreateOr(LHS: Flags, RHS: StrictBlocksFlag);
668 Flags = Builder.CreateOr(LHS: Flags, RHS: StrictThreadsFlag);
669
670 assert(!KernelArgs.NumTeams.empty() && !KernelArgs.NumThreads.empty());
671
672 Value *NumTeams3D =
673 Builder.CreateInsertValue(Agg: ZeroArray, Val: KernelArgs.NumTeams[0], Idxs: {0});
674 Value *NumThreads3D =
675 Builder.CreateInsertValue(Agg: ZeroArray, Val: KernelArgs.NumThreads[0], Idxs: {0});
676 for (unsigned I :
677 seq<unsigned>(Begin: 1, End: std::min(a: KernelArgs.NumTeams.size(), b: MaxDim)))
678 NumTeams3D =
679 Builder.CreateInsertValue(Agg: NumTeams3D, Val: KernelArgs.NumTeams[I], Idxs: {I});
680 for (unsigned I :
681 seq<unsigned>(Begin: 1, End: std::min(a: KernelArgs.NumThreads.size(), b: MaxDim)))
682 NumThreads3D =
683 Builder.CreateInsertValue(Agg: NumThreads3D, Val: KernelArgs.NumThreads[I], Idxs: {I});
684
685 ArgsVector = {Version,
686 PointerNum,
687 KernelArgs.RTArgs.BasePointersArray,
688 KernelArgs.RTArgs.PointersArray,
689 KernelArgs.RTArgs.SizesArray,
690 KernelArgs.RTArgs.MapTypesArray,
691 KernelArgs.RTArgs.MapNamesArray,
692 KernelArgs.RTArgs.MappersArray,
693 KernelArgs.NumIterations,
694 Flags,
695 NumTeams3D,
696 NumThreads3D,
697 KernelArgs.DynCGroupMem};
698}
699
700void OpenMPIRBuilder::addAttributes(omp::RuntimeFunction FnID, Function &Fn) {
701 LLVMContext &Ctx = Fn.getContext();
702
703 // Get the function's current attributes.
704 auto Attrs = Fn.getAttributes();
705 auto FnAttrs = Attrs.getFnAttrs();
706 auto RetAttrs = Attrs.getRetAttrs();
707 SmallVector<AttributeSet, 4> ArgAttrs;
708 for (size_t ArgNo = 0; ArgNo < Fn.arg_size(); ++ArgNo)
709 ArgAttrs.emplace_back(Args: Attrs.getParamAttrs(ArgNo));
710
711 // Add AS to FnAS while taking special care with integer extensions.
712 auto addAttrSet = [&](AttributeSet &FnAS, const AttributeSet &AS,
713 bool Param = true) -> void {
714 bool HasSignExt = AS.hasAttribute(Kind: Attribute::SExt);
715 bool HasZeroExt = AS.hasAttribute(Kind: Attribute::ZExt);
716 if (HasSignExt || HasZeroExt) {
717 assert(AS.getNumAttributes() == 1 &&
718 "Currently not handling extension attr combined with others.");
719 if (Param) {
720 if (auto AK = TargetLibraryInfo::getExtAttrForI32Param(T, Signed: HasSignExt))
721 FnAS = FnAS.addAttribute(C&: Ctx, Kind: AK);
722 } else if (auto AK =
723 TargetLibraryInfo::getExtAttrForI32Return(T, Signed: HasSignExt))
724 FnAS = FnAS.addAttribute(C&: Ctx, Kind: AK);
725 } else {
726 FnAS = FnAS.addAttributes(C&: Ctx, AS);
727 }
728 };
729
730#define OMP_ATTRS_SET(VarName, AttrSet) AttributeSet VarName = AttrSet;
731#include "llvm/Frontend/OpenMP/OMPKinds.def"
732
733 // Add attributes to the function declaration.
734 switch (FnID) {
735#define OMP_RTL_ATTRS(Enum, FnAttrSet, RetAttrSet, ArgAttrSets) \
736 case Enum: \
737 FnAttrs = FnAttrs.addAttributes(Ctx, FnAttrSet); \
738 addAttrSet(RetAttrs, RetAttrSet, /*Param*/ false); \
739 for (size_t ArgNo = 0; ArgNo < ArgAttrSets.size(); ++ArgNo) \
740 addAttrSet(ArgAttrs[ArgNo], ArgAttrSets[ArgNo]); \
741 Fn.setAttributes(AttributeList::get(Ctx, FnAttrs, RetAttrs, ArgAttrs)); \
742 break;
743#include "llvm/Frontend/OpenMP/OMPKinds.def"
744 default:
745 // Attributes are optional.
746 break;
747 }
748}
749
750FunctionCallee
751OpenMPIRBuilder::getOrCreateRuntimeFunction(Module &M, RuntimeFunction FnID) {
752 FunctionType *FnTy = nullptr;
753 StringRef Name;
754
755 switch (FnID) {
756#define OMP_RTL(Enum, Str, IsVarArg, ReturnType, ...) \
757 case Enum: \
758 FnTy = FunctionType::get(ReturnType, ArrayRef<Type *>{__VA_ARGS__}, \
759 IsVarArg); \
760 Name = Str; \
761 break;
762#include "llvm/Frontend/OpenMP/OMPKinds.def"
763 }
764
765 return M.getOrInsertFunction(Name, T: FnTy, InitCallback: [&](Function &Fn) {
766 Fn.setCallingConv(Config.getRuntimeCC());
767 // Add information if the runtime function takes a callback function
768 if (FnID == OMPRTL___kmpc_fork_call || FnID == OMPRTL___kmpc_fork_teams) {
769 LLVMContext &Ctx = Fn.getContext();
770 MDBuilder MDB(Ctx);
771 // Annotate the callback behavior of the runtime function:
772 // - The callback callee is argument number 2 (microtask).
773 // - The first two arguments of the callback callee are unknown (-1).
774 // - All variadic arguments to the runtime function are passed to the
775 // callback callee.
776 Fn.addMetadata(
777 KindID: LLVMContext::MD_callback,
778 MD&: *MDNode::get(Context&: Ctx, MDs: {MDB.createCallbackEncoding(
779 CalleeArgNo: 2, Arguments: {-1, -1}, /* VarArgsArePassed */ true)}));
780 }
781
782 LLVM_DEBUG(dbgs() << "Created OpenMP runtime function " << Fn.getName()
783 << " with type " << *Fn.getFunctionType() << "\n");
784 addAttributes(FnID, Fn);
785 });
786}
787
788Expected<BasicBlock *>
789OpenMPIRBuilder::FinalizationInfo::getFiniBB(IRBuilderBase &Builder) {
790 if (!FiniBB) {
791 Function *ParentFunc = Builder.GetInsertBlock()->getParent();
792 IRBuilderBase::InsertPointGuard Guard(Builder);
793 FiniBB = BasicBlock::Create(Context&: Builder.getContext(), Name: ".fini", Parent: ParentFunc);
794 Builder.SetInsertPoint(FiniBB);
795 // FiniCB adds the branch to the exit stub.
796 if (Error Err = FiniCB(Builder.saveIP()))
797 return Err;
798 }
799 return FiniBB;
800}
801
802Error OpenMPIRBuilder::FinalizationInfo::mergeFiniBB(IRBuilderBase &Builder,
803 BasicBlock *OtherFiniBB) {
804 // Simple case: FiniBB does not exist yet: re-use OtherFiniBB.
805 if (!FiniBB) {
806 FiniBB = OtherFiniBB;
807
808 Builder.SetInsertPoint(FiniBB->getFirstNonPHIIt());
809 if (Error Err = FiniCB(Builder.saveIP()))
810 return Err;
811
812 return Error::success();
813 }
814
815 // Move instructions from FiniBB to the start of OtherFiniBB.
816 auto EndIt = FiniBB->end();
817 if (FiniBB->size() >= 1)
818 if (auto Prev = std::prev(x: EndIt); Prev->isTerminator())
819 EndIt = Prev;
820 OtherFiniBB->splice(ToIt: OtherFiniBB->getFirstNonPHIIt(), FromBB: FiniBB, FromBeginIt: FiniBB->begin(),
821 FromEndIt: EndIt);
822
823 FiniBB->replaceAllUsesWith(V: OtherFiniBB);
824 FiniBB->eraseFromParent();
825 FiniBB = OtherFiniBB;
826 return Error::success();
827}
828
829Function *OpenMPIRBuilder::getOrCreateRuntimeFunctionPtr(RuntimeFunction FnID) {
830 FunctionCallee RTLFn = getOrCreateRuntimeFunction(M, FnID);
831 auto *Fn = dyn_cast<llvm::Function>(Val: RTLFn.getCallee());
832 assert(Fn && "Failed to create OpenMP runtime function pointer");
833 return Fn;
834}
835
836CallInst *OpenMPIRBuilder::createRuntimeFunctionCall(FunctionCallee Callee,
837 ArrayRef<Value *> Args,
838 StringRef Name) {
839 CallInst *Call = Builder.CreateCall(Callee, Args, Name);
840 Call->setCallingConv(Config.getRuntimeCC());
841 return Call;
842}
843
844void OpenMPIRBuilder::initialize() { initializeTypes(M); }
845
846static void raiseUserConstantDataAllocasToEntryBlock(IRBuilderBase &Builder,
847 Function *Function) {
848 BasicBlock &EntryBlock = Function->getEntryBlock();
849 BasicBlock::iterator MoveLocInst = EntryBlock.getFirstNonPHIIt();
850
851 // Loop over blocks looking for constant allocas, skipping the entry block
852 // as any allocas there are already in the desired location.
853 for (auto Block = std::next(x: Function->begin(), n: 1); Block != Function->end();
854 Block++) {
855 for (auto Inst = Block->getReverseIterator()->begin();
856 Inst != Block->getReverseIterator()->end();) {
857 if (auto *AllocaInst = dyn_cast_if_present<llvm::AllocaInst>(Val&: Inst)) {
858 Inst++;
859 if (!isa<ConstantData>(Val: AllocaInst->getArraySize()))
860 continue;
861 AllocaInst->moveBeforePreserving(MovePos: MoveLocInst);
862 } else {
863 Inst++;
864 }
865 }
866 }
867}
868
869static void hoistNonEntryAllocasToEntryBlock(llvm::BasicBlock &Block) {
870 llvm::SmallVector<llvm::Instruction *> AllocasToMove;
871
872 auto ShouldHoistAlloca = [](const llvm::AllocaInst &AllocaInst) {
873 // TODO: For now, we support simple static allocations, we might need to
874 // move non-static ones as well. However, this will need further analysis to
875 // move the lenght arguments as well.
876 return !AllocaInst.isArrayAllocation();
877 };
878
879 for (llvm::Instruction &Inst : Block)
880 if (auto *AllocaInst = llvm::dyn_cast<llvm::AllocaInst>(Val: &Inst))
881 if (ShouldHoistAlloca(*AllocaInst))
882 AllocasToMove.push_back(Elt: AllocaInst);
883
884 auto InsertPoint =
885 Block.getParent()->getEntryBlock().getTerminator()->getIterator();
886
887 for (llvm::Instruction *AllocaInst : AllocasToMove)
888 AllocaInst->moveBefore(InsertPos: InsertPoint);
889}
890
891static void hoistNonEntryAllocasToEntryBlock(llvm::Function *Func) {
892 PostDominatorTree PostDomTree(*Func);
893 for (llvm::BasicBlock &BB : *Func)
894 if (PostDomTree.properlyDominates(A: &BB, B: &Func->getEntryBlock()))
895 hoistNonEntryAllocasToEntryBlock(Block&: BB);
896}
897
898void OpenMPIRBuilder::finalize(Function *Fn) {
899 SmallPtrSet<BasicBlock *, 32> ParallelRegionBlockSet;
900 SmallVector<BasicBlock *, 32> Blocks;
901 SmallVector<std::unique_ptr<OutlineInfo>, 16> DeferredOutlines;
902 for (std::unique_ptr<OutlineInfo> &OI : OutlineInfos) {
903 // Skip functions that have not finalized yet; may happen with nested
904 // function generation.
905 if (Fn && OI->getFunction() != Fn) {
906 DeferredOutlines.push_back(Elt: std::move(OI));
907 continue;
908 }
909
910 ParallelRegionBlockSet.clear();
911 Blocks.clear();
912 OI->collectBlocks(BlockSet&: ParallelRegionBlockSet, BlockVector&: Blocks);
913
914 Function *OuterFn = OI->getFunction();
915 CodeExtractorAnalysisCache CEAC(*OuterFn);
916 // If we generate code for the target device, we need to allocate
917 // struct for aggregate params in the device default alloca address space.
918 // OpenMP runtime requires that the params of the extracted functions are
919 // passed as zero address space pointers. This flag ensures that
920 // CodeExtractor generates correct code for extracted functions
921 // which are used by OpenMP runtime.
922 bool ArgsInZeroAddressSpace = Config.isTargetDevice();
923 std::unique_ptr<CodeExtractor> Extractor =
924 OI->createCodeExtractor(Blocks, ArgsInZeroAddressSpace, Suffix: ".omp_par");
925
926 LLVM_DEBUG(dbgs() << "Before outlining: " << *OuterFn << "\n");
927 LLVM_DEBUG(dbgs() << "Entry " << OI->EntryBB->getName()
928 << " Exit: " << OI->ExitBB->getName() << "\n");
929 assert(Extractor->isEligible() &&
930 "Expected OpenMP outlining to be possible!");
931
932 for (auto *V : OI->ExcludeArgsFromAggregate)
933 Extractor->excludeArgFromAggregate(Arg: V);
934
935 Function *OutlinedFn =
936 Extractor->extractCodeRegion(CEAC, Inputs&: OI->Inputs, Outputs&: OI->Outputs);
937
938 // Forward target-cpu, target-features attributes to the outlined function.
939 auto TargetCpuAttr = OuterFn->getFnAttribute(Kind: "target-cpu");
940 if (TargetCpuAttr.isStringAttribute())
941 OutlinedFn->addFnAttr(Attr: TargetCpuAttr);
942
943 auto TargetFeaturesAttr = OuterFn->getFnAttribute(Kind: "target-features");
944 if (TargetFeaturesAttr.isStringAttribute())
945 OutlinedFn->addFnAttr(Attr: TargetFeaturesAttr);
946
947 LLVM_DEBUG(dbgs() << "After outlining: " << *OuterFn << "\n");
948 LLVM_DEBUG(dbgs() << " Outlined function: " << *OutlinedFn << "\n");
949 assert(OutlinedFn->getReturnType()->isVoidTy() &&
950 "OpenMP outlined functions should not return a value!");
951
952 // For compability with the clang CG we move the outlined function after the
953 // one with the parallel region.
954 OutlinedFn->removeFromParent();
955 M.getFunctionList().insertAfter(where: OuterFn->getIterator(), New: OutlinedFn);
956
957 // Remove the artificial entry introduced by the extractor right away, we
958 // made our own entry block after all.
959 {
960 BasicBlock &ArtificialEntry = OutlinedFn->getEntryBlock();
961 assert(ArtificialEntry.getUniqueSuccessor() == OI->EntryBB);
962 assert(OI->EntryBB->getUniquePredecessor() == &ArtificialEntry);
963 // Move instructions from the to-be-deleted ArtificialEntry to the entry
964 // basic block of the parallel region. CodeExtractor generates
965 // instructions to unwrap the aggregate argument and may sink
966 // allocas/bitcasts for values that are solely used in the outlined region
967 // and do not escape.
968 assert(!ArtificialEntry.empty() &&
969 "Expected instructions to add in the outlined region entry");
970 for (BasicBlock::reverse_iterator It = ArtificialEntry.rbegin(),
971 End = ArtificialEntry.rend();
972 It != End;) {
973 Instruction &I = *It;
974 It++;
975
976 if (I.isTerminator()) {
977 // Absorb any debug value that terminator may have
978 if (Instruction *TI = OI->EntryBB->getTerminatorOrNull())
979 TI->adoptDbgRecords(BB: &ArtificialEntry, It: I.getIterator(), InsertAtHead: false);
980 continue;
981 }
982
983 I.moveBeforePreserving(BB&: *OI->EntryBB,
984 I: OI->EntryBB->getFirstInsertionPt());
985 }
986
987 OI->EntryBB->moveBefore(MovePos: &ArtificialEntry);
988 ArtificialEntry.eraseFromParent();
989 }
990 assert(&OutlinedFn->getEntryBlock() == OI->EntryBB);
991 assert(OutlinedFn && OutlinedFn->hasNUses(1));
992
993 // Run a user callback, e.g. to add attributes.
994 if (OI->PostOutlineCB)
995 OI->PostOutlineCB(*OutlinedFn);
996
997 if (OI->FixUpNonEntryAllocas)
998 hoistNonEntryAllocasToEntryBlock(Func: OutlinedFn);
999 }
1000
1001 // Remove work items that have been completed.
1002 OutlineInfos = std::move(DeferredOutlines);
1003
1004 // The createTarget functions embeds user written code into
1005 // the target region which may inject allocas which need to
1006 // be moved to the entry block of our target or risk malformed
1007 // optimisations by later passes (however, we do not block on
1008 // that here, it's up to the inserter to the list to do so).
1009 // This notbaly has to occur after the OutlinedInfo candidates
1010 // have been extracted so we have an end product that will not
1011 // be implicitly adversely affected by any raises unless
1012 // intentionally appended to the list.
1013 // NOTE: This only does so for ConstantData, it could be extended
1014 // to ConstantExpr's with further effort, however, they should
1015 // largely be folded when they get here. Extending it to runtime
1016 // defined/read+writeable allocation sizes would be non-trivial
1017 // (need to factor in movement of any stores to variables the
1018 // allocation size depends on, as well as the usual loads,
1019 // otherwise it'll yield the wrong result after movement) and
1020 // likely be more suitable as an LLVM optimisation pass.
1021 for (Function *F : ConstantAllocaRaiseCandidates)
1022 raiseUserConstantDataAllocasToEntryBlock(Builder, Function: F);
1023
1024 EmitMetadataErrorReportFunctionTy &&ErrorReportFn =
1025 [](EmitMetadataErrorKind Kind,
1026 const TargetRegionEntryInfo &EntryInfo) -> void {
1027 errs() << "Error of kind: " << Kind
1028 << " when emitting offload entries and metadata during "
1029 "OMPIRBuilder finalization \n";
1030 };
1031
1032 if (!OffloadInfoManager.empty())
1033 createOffloadEntriesAndInfoMetadata(ErrorReportFunction&: ErrorReportFn);
1034
1035 // Rewrite uses of globals to their replacement declare target globals if
1036 // we are processing a device module.
1037 if (Config.isTargetDevice())
1038 applyDeclareTargetGlobalReplacements();
1039
1040 if (Config.EmitLLVMUsedMetaInfo.value_or(u: false)) {
1041 std::vector<WeakTrackingVH> LLVMCompilerUsed = {
1042 M.getGlobalVariable(Name: "__openmp_nvptx_data_transfer_temporary_storage")};
1043 emitUsed(Name: "llvm.compiler.used", List: LLVMCompilerUsed);
1044 }
1045
1046 IsFinalized = true;
1047}
1048
1049bool OpenMPIRBuilder::isFinalized() { return IsFinalized; }
1050
1051void OpenMPIRBuilder::registerDeclareTargetGlobalReplacement(
1052 GlobalValue *Original, GlobalValue *Replacement) {
1053 assert(Original && Replacement &&
1054 "Null values provided to registerDeclareTargetGlobalReplacement");
1055 DeclareTargetGlobalReplacements.push_back(Elt: {.Original: Original, .Replacement: Replacement});
1056}
1057
1058void OpenMPIRBuilder::applyDeclareTargetGlobalReplacements() {
1059 for (DeclareTargetGlobalReplacement &R : DeclareTargetGlobalReplacements) {
1060 GlobalValue *OldGV = R.Original;
1061 GlobalValue *NewGV = R.Replacement;
1062
1063 assert(OldGV && NewGV &&
1064 "A null value was inserted into DeclareTargetGlobalReplacements");
1065
1066 // The assert above should catch this case, but this is kept to attempt
1067 // to proceed without issue when asserts are off.
1068 if (!OldGV || !NewGV)
1069 continue;
1070
1071 // The replacement global is a reference pointer that holds the
1072 // address of the device-resident storage. Every use must load the
1073 // reference pointer first and use the loaded address.
1074 //
1075 // Constant expression users (e.g. a constant GEP embedded in another
1076 // global's initializer or in an instruction) cannot have a load inserted
1077 // in place, so first expand any constant-expression users that live inside
1078 // functions into instructions. Any remaining constant users are handled
1079 // via a direct constant rewrite below as we cannot materialize a load
1080 // there.
1081 //
1082 // NOTE: We extend the constant rewrite to module scope, as we replace all
1083 // usages.
1084 if (auto *OldConst = dyn_cast<Constant>(Val: OldGV))
1085 convertUsersOfConstantsToInstructions(Consts: OldConst,
1086 /*RestrictToFunc=*/nullptr,
1087 /*RemoveDeadConstants=*/false);
1088
1089 IRBuilderBase::InsertPointGuard Guard(Builder);
1090 SmallVector<User *, 16> Users(OldGV->users());
1091 for (User *U : Users) {
1092 auto *Insn = dyn_cast<Instruction>(Val: U);
1093 if (!Insn)
1094 continue;
1095
1096 // A PHI node cannot have a load inserted immediately before it, as PHIs
1097 // must remain grouped at the top of their basic block. So we need to
1098 // make sure any loads we emit are generated in the preceding edge, a
1099 // PHI may reference the global on more than one edge, so every matching
1100 // slot must be handled.
1101 if (auto *PHI = dyn_cast<PHINode>(Val: Insn)) {
1102 for (unsigned I = 0, E = PHI->getNumIncomingValues(); I < E; ++I) {
1103 if (PHI->getIncomingValue(i: I) != OldGV)
1104 continue;
1105
1106 BasicBlock *IncomingBB = PHI->getIncomingBlock(i: I);
1107 Builder.SetInsertPoint(IncomingBB->getTerminator());
1108 Builder.SetCurrentDebugLocation(PHI->getDebugLoc());
1109 LoadInst *EdgeLoad = Builder.CreateLoad(Ty: NewGV->getType(), Ptr: NewGV);
1110 PHI->setIncomingValue(i: I, V: EdgeLoad);
1111 }
1112 continue;
1113 }
1114
1115 Builder.SetInsertPoint(Insn);
1116 Builder.SetCurrentDebugLocation(Insn->getDebugLoc());
1117 LoadInst *Load = Builder.CreateLoad(Ty: NewGV->getType(), Ptr: NewGV);
1118
1119 // The replacement declare target global lives in the default address
1120 // space, whereas the original global may reside in a non-default
1121 // address space. In that case the initial lowering may have
1122 // emitted an addrspacecast that is no longer valid. Replace the
1123 // whole addrspacecast with the load and erase it rather than
1124 // feeding the load back into the (now pointless) cast.
1125 // NOTE: If we end up with replacement declare target globals in
1126 // non-zero AS's the below will need some minor extensions to have the
1127 // option to alter the address space cast to the new address space where
1128 // required rather than just replacing it.
1129 if (auto *ASC = dyn_cast<AddrSpaceCastInst>(Val: Insn)) {
1130 unsigned NewGVAS = NewGV->getType()->getPointerAddressSpace();
1131 assert(NewGVAS == 0 &&
1132 "Non-default address space declare target global");
1133 unsigned OldGVAS = OldGV->getType()->getPointerAddressSpace();
1134 unsigned DestAS = ASC->getType()->getPointerAddressSpace();
1135 if (DestAS == 0 && NewGVAS != OldGVAS) {
1136 ASC->replaceAllUsesWith(V: Load);
1137 ASC->eraseFromParent();
1138 continue;
1139 }
1140 }
1141
1142 Insn->replaceUsesOfWith(From: OldGV, To: Load);
1143 }
1144 }
1145
1146 DeclareTargetGlobalReplacements.clear();
1147}
1148
1149OpenMPIRBuilder::~OpenMPIRBuilder() {
1150 assert(OutlineInfos.empty() && "There must be no outstanding outlinings");
1151}
1152
1153GlobalValue *OpenMPIRBuilder::createGlobalFlag(unsigned Value, StringRef Name) {
1154 IntegerType *I32Ty = Type::getInt32Ty(C&: M.getContext());
1155 auto *GV =
1156 new GlobalVariable(M, I32Ty,
1157 /* isConstant = */ true, GlobalValue::WeakODRLinkage,
1158 ConstantInt::get(Ty: I32Ty, V: Value), Name);
1159 GV->setVisibility(GlobalValue::HiddenVisibility);
1160
1161 return GV;
1162}
1163
1164void OpenMPIRBuilder::emitUsed(StringRef Name, ArrayRef<WeakTrackingVH> List) {
1165 if (List.empty())
1166 return;
1167
1168 // Convert List to what ConstantArray needs.
1169 SmallVector<Constant *, 8> UsedArray;
1170 UsedArray.resize(N: List.size());
1171 for (unsigned I = 0, E = List.size(); I != E; ++I)
1172 UsedArray[I] = ConstantExpr::getPointerBitCastOrAddrSpaceCast(
1173 C: cast<Constant>(Val: &*List[I]), Ty: Builder.getPtrTy());
1174
1175 if (UsedArray.empty())
1176 return;
1177 ArrayType *ATy = ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: UsedArray.size());
1178
1179 auto *GV = new GlobalVariable(M, ATy, false, GlobalValue::AppendingLinkage,
1180 ConstantArray::get(T: ATy, V: UsedArray), Name);
1181
1182 GV->setSection("llvm.metadata");
1183}
1184
1185GlobalVariable *
1186OpenMPIRBuilder::emitKernelExecutionMode(StringRef KernelName,
1187 OMPTgtExecModeFlags Mode) {
1188 auto *Int8Ty = Builder.getInt8Ty();
1189 auto *GVMode = new GlobalVariable(
1190 M, Int8Ty, /*isConstant=*/true, GlobalValue::WeakAnyLinkage,
1191 ConstantInt::get(Ty: Int8Ty, V: Mode), Twine(KernelName, "_exec_mode"));
1192 GVMode->setVisibility(GlobalVariable::ProtectedVisibility);
1193 return GVMode;
1194}
1195
1196Constant *OpenMPIRBuilder::getOrCreateIdent(Constant *SrcLocStr,
1197 uint32_t SrcLocStrSize,
1198 IdentFlag LocFlags,
1199 unsigned Reserve2Flags) {
1200 // Enable "C-mode".
1201 LocFlags |= OMP_IDENT_FLAG_KMPC;
1202
1203 Constant *&Ident =
1204 IdentMap[{SrcLocStr, uint64_t(LocFlags) << 31 | Reserve2Flags}];
1205 if (!Ident) {
1206 Constant *I32Null = ConstantInt::getNullValue(Ty: Int32);
1207 Constant *IdentData[] = {I32Null,
1208 ConstantInt::get(Ty: Int32, V: uint32_t(LocFlags)),
1209 ConstantInt::get(Ty: Int32, V: Reserve2Flags),
1210 ConstantInt::get(Ty: Int32, V: SrcLocStrSize), SrcLocStr};
1211
1212 size_t SrcLocStrArgIdx = 4;
1213 if (OpenMPIRBuilder::Ident->getElementType(N: SrcLocStrArgIdx)
1214 ->getPointerAddressSpace() !=
1215 IdentData[SrcLocStrArgIdx]->getType()->getPointerAddressSpace())
1216 IdentData[SrcLocStrArgIdx] = ConstantExpr::getAddrSpaceCast(
1217 C: SrcLocStr, Ty: OpenMPIRBuilder::Ident->getElementType(N: SrcLocStrArgIdx));
1218 Constant *Initializer =
1219 ConstantStruct::get(T: OpenMPIRBuilder::Ident, V: IdentData);
1220
1221 // Look for existing encoding of the location + flags, not needed but
1222 // minimizes the difference to the existing solution while we transition.
1223 for (GlobalVariable &GV : M.globals())
1224 if (GV.getValueType() == OpenMPIRBuilder::Ident && GV.hasInitializer())
1225 if (GV.getInitializer() == Initializer)
1226 Ident = &GV;
1227
1228 if (!Ident) {
1229 auto *GV = new GlobalVariable(
1230 M, OpenMPIRBuilder::Ident,
1231 /* isConstant = */ true, GlobalValue::PrivateLinkage, Initializer, "",
1232 nullptr, GlobalValue::NotThreadLocal,
1233 M.getDataLayout().getDefaultGlobalsAddressSpace());
1234 GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
1235 GV->setAlignment(Align(8));
1236 Ident = GV;
1237 }
1238 }
1239
1240 return ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: Ident, Ty: IdentPtr);
1241}
1242
1243Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(StringRef LocStr,
1244 uint32_t &SrcLocStrSize) {
1245 SrcLocStrSize = LocStr.size();
1246 Constant *&SrcLocStr = SrcLocStrMap[LocStr];
1247 if (!SrcLocStr) {
1248 Constant *Initializer =
1249 ConstantDataArray::getString(Context&: M.getContext(), Initializer: LocStr);
1250
1251 // Look for existing encoding of the location, not needed but minimizes the
1252 // difference to the existing solution while we transition.
1253 for (GlobalVariable &GV : M.globals())
1254 if (GV.isConstant() && GV.hasInitializer() &&
1255 GV.getInitializer() == Initializer)
1256 return SrcLocStr = ConstantExpr::getPointerCast(C: &GV, Ty: Int8Ptr);
1257
1258 SrcLocStr = Builder.CreateGlobalString(
1259 Str: LocStr, /*Name=*/"", AddressSpace: M.getDataLayout().getDefaultGlobalsAddressSpace(),
1260 M: &M);
1261 }
1262 return SrcLocStr;
1263}
1264
1265Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(StringRef FunctionName,
1266 StringRef FileName,
1267 unsigned Line, unsigned Column,
1268 uint32_t &SrcLocStrSize) {
1269 SmallString<128> Buffer;
1270 Buffer.push_back(Elt: ';');
1271 Buffer.append(RHS: FileName);
1272 Buffer.push_back(Elt: ';');
1273 Buffer.append(RHS: FunctionName);
1274 Buffer.push_back(Elt: ';');
1275 Buffer.append(RHS: std::to_string(val: Line));
1276 Buffer.push_back(Elt: ';');
1277 Buffer.append(RHS: std::to_string(val: Column));
1278 Buffer.push_back(Elt: ';');
1279 Buffer.push_back(Elt: ';');
1280 return getOrCreateSrcLocStr(LocStr: Buffer.str(), SrcLocStrSize);
1281}
1282
1283Constant *
1284OpenMPIRBuilder::getOrCreateDefaultSrcLocStr(uint32_t &SrcLocStrSize) {
1285 StringRef UnknownLoc = ";unknown;unknown;0;0;;";
1286 return getOrCreateSrcLocStr(LocStr: UnknownLoc, SrcLocStrSize);
1287}
1288
1289Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(DebugLoc DL,
1290 uint32_t &SrcLocStrSize,
1291 const Function *F) {
1292 DILocation *DIL = DL.get();
1293 if (!DIL)
1294 return getOrCreateDefaultSrcLocStr(SrcLocStrSize);
1295 StringRef FileName =
1296 !DIL->getFilename().empty() ? DIL->getFilename() : M.getName();
1297 StringRef Function = DIL->getScope()->getSubprogram()->getName();
1298 if (Function.empty() && F)
1299 Function = F->getName();
1300 return getOrCreateSrcLocStr(FunctionName: Function, FileName, Line: DIL->getLine(),
1301 Column: DIL->getColumn(), SrcLocStrSize);
1302}
1303
1304Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(const LocationDescription &Loc,
1305 uint32_t &SrcLocStrSize) {
1306 return getOrCreateSrcLocStr(DL: Loc.DL, SrcLocStrSize,
1307 F: Loc.IP.getNodeParent()->getParent());
1308}
1309
1310Value *OpenMPIRBuilder::getOrCreateThreadID(Value *Ident) {
1311 return createRuntimeFunctionCall(
1312 Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_global_thread_num), Args: Ident,
1313 Name: "omp_global_thread_num");
1314}
1315
1316OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTargetInReduction(
1317 const LocationDescription &Loc, ArrayRef<Value *> OrigPtrs,
1318 ArrayRef<Type *> ResultPtrTys,
1319 function_ref<void(unsigned, Value *)> MapPrivateCB) {
1320 assert(OrigPtrs.size() == ResultPtrTys.size() &&
1321 "expected one result pointer type per in_reduction item");
1322 if (!updateToLocation(Loc))
1323 return Loc.IP;
1324 if (OrigPtrs.empty())
1325 return Builder.saveIP();
1326
1327 // Compute the executing thread's gtid once for the whole target body and
1328 // reuse it for every in_reduction lookup, so a target with several
1329 // in_reduction items does not emit a redundant __kmpc_global_thread_num per
1330 // item.
1331 uint32_t SrcLocStrSize;
1332 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1333 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
1334 Value *Gtid = getOrCreateThreadID(Ident);
1335
1336 // The runtime entry point takes (and returns) a generic, default-address-
1337 // space `ptr`. A NULL descriptor makes the runtime walk the enclosing
1338 // taskgroups to find the matching task_reduction registration for the item.
1339 Type *PtrTy = PointerType::getUnqual(C&: M.getContext());
1340 Value *NullDesc = ConstantPointerNull::get(T: PtrTy);
1341 FunctionCallee GetThData =
1342 getOrCreateRuntimeFunction(M, FnID: OMPRTL___kmpc_task_reduction_get_th_data);
1343
1344 for (unsigned Idx = 0; Idx < OrigPtrs.size(); ++Idx) {
1345 // Normalize a non-default-address-space original pointer to the generic
1346 // address space before the call.
1347 Value *OrigPtr = OrigPtrs[Idx];
1348 if (auto *OrigPtrTy = dyn_cast<PointerType>(Val: OrigPtr->getType());
1349 OrigPtrTy && OrigPtrTy->getAddressSpace() != 0)
1350 OrigPtr = Builder.CreateAddrSpaceCast(V: OrigPtr, DestTy: PtrTy);
1351
1352 Value *Priv = Builder.CreateCall(Callee: GetThData, Args: {Gtid, NullDesc, OrigPtr},
1353 Name: "omp.inred.priv");
1354
1355 // Cast the returned private pointer back to the requested address space
1356 // when it differs.
1357 if (auto *ResPtrTy = dyn_cast<PointerType>(Val: ResultPtrTys[Idx]);
1358 ResPtrTy && ResPtrTy->getAddressSpace() != 0)
1359 Priv = Builder.CreateAddrSpaceCast(V: Priv, DestTy: ResultPtrTys[Idx]);
1360
1361 MapPrivateCB(Idx, Priv);
1362 }
1363 return Builder.saveIP();
1364}
1365
1366OpenMPIRBuilder::InsertPointOrErrorTy
1367OpenMPIRBuilder::createBarrier(const LocationDescription &Loc, Directive Kind,
1368 bool ForceSimpleCall, bool CheckCancelFlag) {
1369 if (!updateToLocation(Loc))
1370 return Loc.IP;
1371
1372 // Build call __kmpc_cancel_barrier(loc, thread_id) or
1373 // __kmpc_barrier(loc, thread_id);
1374
1375 IdentFlag BarrierLocFlags;
1376 switch (Kind) {
1377 case OMPD_for:
1378 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_FOR;
1379 break;
1380 case OMPD_sections:
1381 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_SECTIONS;
1382 break;
1383 case OMPD_single:
1384 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_SINGLE;
1385 break;
1386 case OMPD_barrier:
1387 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_EXPL;
1388 break;
1389 default:
1390 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL;
1391 break;
1392 }
1393
1394 uint32_t SrcLocStrSize;
1395 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1396 Value *Args[] = {
1397 getOrCreateIdent(SrcLocStr, SrcLocStrSize, LocFlags: BarrierLocFlags),
1398 getOrCreateThreadID(Ident: getOrCreateIdent(SrcLocStr, SrcLocStrSize))};
1399
1400 // If we are in a cancellable parallel region, barriers are cancellation
1401 // points.
1402 // TODO: Check why we would force simple calls or to ignore the cancel flag.
1403 bool UseCancelBarrier =
1404 !ForceSimpleCall && isLastFinalizationInfoCancellable(DK: OMPD_parallel);
1405
1406 Value *Result = createRuntimeFunctionCall(
1407 Callee: getOrCreateRuntimeFunctionPtr(FnID: UseCancelBarrier
1408 ? OMPRTL___kmpc_cancel_barrier
1409 : OMPRTL___kmpc_barrier),
1410 Args);
1411
1412 if (UseCancelBarrier && CheckCancelFlag)
1413 if (Error Err = emitCancelationCheckImpl(CancelFlag: Result, CanceledDirective: OMPD_parallel))
1414 return Err;
1415
1416 return Builder.saveIP();
1417}
1418
1419OpenMPIRBuilder::InsertPointOrErrorTy
1420OpenMPIRBuilder::createCancel(const LocationDescription &Loc,
1421 Value *IfCondition,
1422 omp::Directive CanceledDirective) {
1423 if (!updateToLocation(Loc))
1424 return Loc.IP;
1425
1426 // LLVM utilities like blocks with terminators.
1427 auto *UI = Builder.CreateUnreachable();
1428
1429 Instruction *ThenTI = UI, *ElseTI = nullptr;
1430 if (IfCondition) {
1431 SplitBlockAndInsertIfThenElse(Cond: IfCondition, SplitBefore: UI, ThenTerm: &ThenTI, ElseTerm: &ElseTI);
1432
1433 // Even if the if condition evaluates to false, this should count as a
1434 // cancellation point
1435 Builder.SetInsertPoint(ElseTI);
1436 auto ElseIP = Builder.saveIP();
1437
1438 InsertPointOrErrorTy IPOrErr = createCancellationPoint(
1439 Loc: LocationDescription{ElseIP, Loc.DL}, CanceledDirective);
1440 if (!IPOrErr)
1441 return IPOrErr;
1442 }
1443
1444 Builder.SetInsertPoint(ThenTI);
1445
1446 Value *CancelKind = nullptr;
1447 switch (CanceledDirective) {
1448#define OMP_CANCEL_KIND(Enum, Str, DirectiveEnum, Value) \
1449 case DirectiveEnum: \
1450 CancelKind = Builder.getInt32(Value); \
1451 break;
1452#include "llvm/Frontend/OpenMP/OMPKinds.def"
1453 default:
1454 llvm_unreachable("Unknown cancel kind!");
1455 }
1456
1457 uint32_t SrcLocStrSize;
1458 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1459 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
1460 Value *Args[] = {Ident, getOrCreateThreadID(Ident), CancelKind};
1461 Value *Result = createRuntimeFunctionCall(
1462 Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_cancel), Args);
1463
1464 // The actual cancel logic is shared with others, e.g., cancel_barriers.
1465 if (Error Err = emitCancelationCheckImpl(CancelFlag: Result, CanceledDirective))
1466 return Err;
1467
1468 // Update the insertion point and remove the terminator we introduced.
1469 Builder.SetInsertPoint(UI->getParent());
1470 UI->eraseFromParent();
1471
1472 return Builder.saveIP();
1473}
1474
1475OpenMPIRBuilder::InsertPointOrErrorTy
1476OpenMPIRBuilder::createCancellationPoint(const LocationDescription &Loc,
1477 omp::Directive CanceledDirective) {
1478 if (!updateToLocation(Loc))
1479 return Loc.IP;
1480
1481 // LLVM utilities like blocks with terminators.
1482 auto *UI = Builder.CreateUnreachable();
1483 Builder.SetInsertPoint(UI);
1484
1485 Value *CancelKind = nullptr;
1486 switch (CanceledDirective) {
1487#define OMP_CANCEL_KIND(Enum, Str, DirectiveEnum, Value) \
1488 case DirectiveEnum: \
1489 CancelKind = Builder.getInt32(Value); \
1490 break;
1491#include "llvm/Frontend/OpenMP/OMPKinds.def"
1492 default:
1493 llvm_unreachable("Unknown cancel kind!");
1494 }
1495
1496 uint32_t SrcLocStrSize;
1497 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1498 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
1499 Value *Args[] = {Ident, getOrCreateThreadID(Ident), CancelKind};
1500 Value *Result = createRuntimeFunctionCall(
1501 Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_cancellationpoint), Args);
1502
1503 // The actual cancel logic is shared with others, e.g., cancel_barriers.
1504 if (Error Err = emitCancelationCheckImpl(CancelFlag: Result, CanceledDirective))
1505 return Err;
1506
1507 // Update the insertion point and remove the terminator we introduced.
1508 Builder.SetInsertPoint(UI->getParent());
1509 UI->eraseFromParent();
1510
1511 return Builder.saveIP();
1512}
1513
1514OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitTargetKernel(
1515 const LocationDescription &Loc, InsertPointTy AllocaIP, Value *&Return,
1516 Value *Ident, Value *DeviceID, Value *NumTeams, Value *NumThreads,
1517 Value *HostPtr, ArrayRef<Value *> KernelArgs) {
1518 if (!updateToLocation(Loc))
1519 return Loc.IP;
1520
1521 Builder.restoreIP(IP: AllocaIP);
1522 auto *KernelArgsPtr =
1523 Builder.CreateAlloca(Ty: OpenMPIRBuilder::KernelArgs, ArraySize: nullptr, Name: "kernel_args");
1524 updateToLocation(Loc);
1525
1526 for (unsigned I = 0, Size = KernelArgs.size(); I != Size; ++I) {
1527 llvm::Value *Arg =
1528 Builder.CreateStructGEP(Ty: OpenMPIRBuilder::KernelArgs, Ptr: KernelArgsPtr, Idx: I);
1529 Builder.CreateAlignedStore(
1530 Val: KernelArgs[I], Ptr: Arg,
1531 Align: M.getDataLayout().getPrefTypeAlign(Ty: KernelArgs[I]->getType()));
1532 }
1533
1534 SmallVector<Value *> OffloadingArgs{Ident, DeviceID, NumTeams,
1535 NumThreads, HostPtr, KernelArgsPtr};
1536
1537 Return = createRuntimeFunctionCall(
1538 Callee: getOrCreateRuntimeFunction(M, FnID: OMPRTL___tgt_target_kernel),
1539 Args: OffloadingArgs);
1540
1541 return Builder.saveIP();
1542}
1543
1544OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitKernelLaunch(
1545 const LocationDescription &Loc, Value *OutlinedFnID,
1546 EmitFallbackCallbackTy EmitTargetCallFallbackCB, TargetKernelArgs &Args,
1547 Value *DeviceID, Value *RTLoc, InsertPointTy AllocaIP) {
1548
1549 if (!updateToLocation(Loc))
1550 return Loc.IP;
1551
1552 // On top of the arrays that were filled up, the target offloading call
1553 // takes as arguments the device id as well as the host pointer. The host
1554 // pointer is used by the runtime library to identify the current target
1555 // region, so it only has to be unique and not necessarily point to
1556 // anything. It could be the pointer to the outlined function that
1557 // implements the target region, but we aren't using that so that the
1558 // compiler doesn't need to keep that, and could therefore inline the host
1559 // function if proven worthwhile during optimization.
1560
1561 // From this point on, we need to have an ID of the target region defined.
1562 assert(OutlinedFnID && "Invalid outlined function ID!");
1563 (void)OutlinedFnID;
1564
1565 // Return value of the runtime offloading call.
1566 Value *Return = nullptr;
1567
1568 // Arguments for the target kernel.
1569 SmallVector<Value *> ArgsVector;
1570 getKernelArgsVector(KernelArgs&: Args, Builder, ArgsVector);
1571
1572 // The target region is an outlined function launched by the runtime
1573 // via calls to __tgt_target_kernel().
1574 //
1575 // Note that on the host and CPU targets, the runtime implementation of
1576 // these calls simply call the outlined function without forking threads.
1577 // The outlined functions themselves have runtime calls to
1578 // __kmpc_fork_teams() and __kmpc_fork() for this purpose, codegen'd by
1579 // the compiler in emitTeamsCall() and emitParallelCall().
1580 //
1581 // In contrast, on the NVPTX target, the implementation of
1582 // __tgt_target_teams() launches a GPU kernel with the requested number
1583 // of teams and threads so no additional calls to the runtime are required.
1584 // Check the error code and execute the host version if required.
1585 Builder.restoreIP(IP: emitTargetKernel(
1586 Loc: Builder, AllocaIP, Return, Ident: RTLoc, DeviceID, NumTeams: Args.NumTeams.front(),
1587 NumThreads: Args.NumThreads.front(), HostPtr: OutlinedFnID, KernelArgs: ArgsVector));
1588
1589 BasicBlock *OffloadFailedBlock =
1590 BasicBlock::Create(Context&: Builder.getContext(), Name: "omp_offload.failed");
1591 BasicBlock *OffloadContBlock =
1592 BasicBlock::Create(Context&: Builder.getContext(), Name: "omp_offload.cont");
1593 Value *Failed = Builder.CreateIsNotNull(Arg: Return);
1594 Builder.CreateCondBr(Cond: Failed, True: OffloadFailedBlock, False: OffloadContBlock);
1595
1596 auto CurFn = Builder.GetInsertBlock()->getParent();
1597 emitBlock(BB: OffloadFailedBlock, CurFn);
1598 InsertPointOrErrorTy AfterIP = EmitTargetCallFallbackCB(Builder.saveIP());
1599 if (!AfterIP)
1600 return AfterIP.takeError();
1601 Builder.restoreIP(IP: *AfterIP);
1602 emitBranch(Target: OffloadContBlock);
1603 emitBlock(BB: OffloadContBlock, CurFn, /*IsFinished=*/true);
1604 return Builder.saveIP();
1605}
1606
1607Error OpenMPIRBuilder::emitCancelationCheckImpl(
1608 Value *CancelFlag, omp::Directive CanceledDirective) {
1609 assert(isLastFinalizationInfoCancellable(CanceledDirective) &&
1610 "Unexpected cancellation!");
1611
1612 // For a cancel barrier we create two new blocks.
1613 BasicBlock *BB = Builder.GetInsertBlock();
1614 BasicBlock *NonCancellationBlock;
1615 if (Builder.GetInsertPoint() == BB->end()) {
1616 // TODO: This branch will not be needed once we moved to the
1617 // OpenMPIRBuilder codegen completely.
1618 NonCancellationBlock = BasicBlock::Create(
1619 Context&: BB->getContext(), Name: BB->getName() + ".cont", Parent: BB->getParent());
1620 } else {
1621 NonCancellationBlock = SplitBlock(Old: BB, SplitPt: &*Builder.GetInsertPoint());
1622 BB->getTerminator()->eraseFromParent();
1623 Builder.SetInsertPoint(BB);
1624 }
1625 BasicBlock *CancellationBlock = BasicBlock::Create(
1626 Context&: BB->getContext(), Name: BB->getName() + ".cncl", Parent: BB->getParent());
1627
1628 // Jump to them based on the return value.
1629 Value *Cmp = Builder.CreateIsNull(Arg: CancelFlag);
1630 Builder.CreateCondBr(Cond: Cmp, True: NonCancellationBlock, False: CancellationBlock,
1631 /* TODO weight */ BranchWeights: nullptr, Unpredictable: nullptr);
1632
1633 // From the cancellation block we finalize all variables and go to the
1634 // post finalization block that is known to the FiniCB callback.
1635 auto &FI = FinalizationStack.back();
1636 Expected<BasicBlock *> FiniBBOrErr = FI.getFiniBB(Builder);
1637 if (!FiniBBOrErr)
1638 return FiniBBOrErr.takeError();
1639 Builder.SetInsertPoint(CancellationBlock);
1640 Builder.CreateBr(Dest: *FiniBBOrErr);
1641
1642 // The continuation block is where code generation continues.
1643 Builder.SetInsertPoint(NonCancellationBlock->begin());
1644 return Error::success();
1645}
1646
1647/// Create wrapper function used to gather the outlined function's argument
1648/// structure from a shared buffer and to forward them to it when running in
1649/// Generic mode.
1650///
1651/// The outlined function is expected to receive 2 integer arguments followed by
1652/// an optional pointer argument to an argument structure holding the rest.
1653static Function *createTargetParallelWrapper(OpenMPIRBuilder *OMPIRBuilder,
1654 Function &OutlinedFn) {
1655 size_t NumArgs = OutlinedFn.arg_size();
1656 assert((NumArgs == 2 || NumArgs == 3) &&
1657 "expected a 2-3 argument parallel outlined function");
1658 bool UseArgStruct = NumArgs == 3;
1659
1660 IRBuilder<> &Builder = OMPIRBuilder->Builder;
1661 IRBuilder<>::InsertPointGuard IPG(Builder);
1662 auto *FnTy = FunctionType::get(Result: Builder.getVoidTy(),
1663 Params: {Builder.getInt16Ty(), Builder.getInt32Ty()},
1664 /*isVarArg=*/false);
1665 auto *WrapperFn =
1666 Function::Create(Ty: FnTy, Linkage: GlobalValue::InternalLinkage,
1667 N: OutlinedFn.getName() + ".wrapper", M&: OMPIRBuilder->M);
1668
1669 WrapperFn->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
1670 WrapperFn->addParamAttr(ArgNo: 0, Kind: Attribute::ZExt);
1671 WrapperFn->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
1672
1673 BasicBlock *EntryBB =
1674 BasicBlock::Create(Context&: OMPIRBuilder->M.getContext(), Name: "entry", Parent: WrapperFn);
1675 Builder.SetInsertPoint(EntryBB);
1676
1677 // Allocation.
1678 Value *AddrAlloca = Builder.CreateAlloca(Ty: Builder.getInt32Ty(),
1679 /*ArraySize=*/nullptr, Name: "addr");
1680 AddrAlloca = Builder.CreatePointerBitCastOrAddrSpaceCast(
1681 V: AddrAlloca, DestTy: Builder.getPtrTy(/*AddrSpace=*/0),
1682 Name: AddrAlloca->getName() + ".ascast");
1683
1684 Value *ZeroAlloca = Builder.CreateAlloca(Ty: Builder.getInt32Ty(),
1685 /*ArraySize=*/nullptr, Name: "zero");
1686 ZeroAlloca = Builder.CreatePointerBitCastOrAddrSpaceCast(
1687 V: ZeroAlloca, DestTy: Builder.getPtrTy(/*AddrSpace=*/0),
1688 Name: ZeroAlloca->getName() + ".ascast");
1689
1690 Value *ArgsAlloca = nullptr;
1691 if (UseArgStruct) {
1692 ArgsAlloca = Builder.CreateAlloca(Ty: Builder.getPtrTy(),
1693 /*ArraySize=*/nullptr, Name: "global_args");
1694 ArgsAlloca = Builder.CreatePointerBitCastOrAddrSpaceCast(
1695 V: ArgsAlloca, DestTy: Builder.getPtrTy(/*AddrSpace=*/0),
1696 Name: ArgsAlloca->getName() + ".ascast");
1697 }
1698
1699 // Initialization.
1700 Builder.CreateStore(Val: WrapperFn->getArg(i: 1), Ptr: AddrAlloca);
1701 Builder.CreateStore(Val: Builder.getInt32(C: 0), Ptr: ZeroAlloca);
1702 if (UseArgStruct) {
1703 Builder.CreateCall(
1704 Callee: OMPIRBuilder->getOrCreateRuntimeFunctionPtr(
1705 FnID: llvm::omp::RuntimeFunction::OMPRTL___kmpc_get_shared_variables),
1706 Args: {ArgsAlloca});
1707 }
1708
1709 SmallVector<Value *, 3> Args{AddrAlloca, ZeroAlloca};
1710
1711 // Load structArg from global_args.
1712 if (UseArgStruct) {
1713 Value *StructArg = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ArgsAlloca);
1714 StructArg = Builder.CreateInBoundsGEP(Ty: Builder.getPtrTy(), Ptr: StructArg,
1715 IdxList: {Builder.getInt64(C: 0)});
1716 StructArg = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: StructArg, Name: "structArg");
1717 Args.push_back(Elt: StructArg);
1718 }
1719
1720 // Call the outlined function holding the parallel body.
1721 Builder.CreateCall(Callee: &OutlinedFn, Args);
1722 Builder.CreateRetVoid();
1723
1724 return WrapperFn;
1725}
1726
1727// Callback used to create OpenMP runtime calls to support
1728// omp parallel clause for the device.
1729// We need to use this callback to replace call to the OutlinedFn in OuterFn
1730// by the call to the OpenMP DeviceRTL runtime function (kmpc_parallel_60)
1731static void targetParallelCallback(
1732 OpenMPIRBuilder *OMPIRBuilder, Function &OutlinedFn, Function *OuterFn,
1733 BasicBlock *OuterAllocaBB, Value *Ident, Value *IfCondition,
1734 Value *NumThreads, Instruction *PrivTID, AllocaInst *PrivTIDAddr,
1735 Value *ThreadID, const SmallVector<Instruction *, 4> &ToBeDeleted) {
1736 assert(OutlinedFn.arg_size() >= 2 &&
1737 "Expected at least tid and bounded tid as arguments");
1738 unsigned NumCapturedVars = OutlinedFn.arg_size() - /* tid & bounded tid */ 2;
1739
1740 // Add some known attributes.
1741 IRBuilder<> &Builder = OMPIRBuilder->Builder;
1742 OutlinedFn.addParamAttr(ArgNo: 0, Kind: Attribute::NoAlias);
1743 OutlinedFn.addParamAttr(ArgNo: 1, Kind: Attribute::NoAlias);
1744 OutlinedFn.addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
1745 OutlinedFn.addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
1746 OutlinedFn.addFnAttr(Kind: Attribute::NoUnwind);
1747
1748 CallInst *CI = cast<CallInst>(Val: OutlinedFn.user_back());
1749 assert(CI && "Expected call instruction to outlined function");
1750 CI->getParent()->setName("omp_parallel");
1751
1752 Builder.SetInsertPoint(CI);
1753 Type *PtrTy = OMPIRBuilder->VoidPtr;
1754
1755 // Add alloca for kernel args
1756 OpenMPIRBuilder ::InsertPointTy CurrentIP = Builder.saveIP();
1757 Builder.SetInsertPoint(OuterAllocaBB->getFirstInsertionPt());
1758 AllocaInst *ArgsAlloca =
1759 Builder.CreateAlloca(Ty: ArrayType::get(ElementType: PtrTy, NumElements: NumCapturedVars));
1760 Value *Args = ArgsAlloca;
1761 // Add address space cast if array for storing arguments is not allocated
1762 // in address space 0
1763 if (ArgsAlloca->getAddressSpace())
1764 Args = Builder.CreatePointerCast(V: ArgsAlloca, DestTy: PtrTy);
1765 Builder.restoreIP(IP: CurrentIP);
1766
1767 // Store captured vars which are used by kmpc_parallel_60
1768 for (unsigned Idx = 0; Idx < NumCapturedVars; Idx++) {
1769 Value *V = *(CI->arg_begin() + 2 + Idx);
1770 Value *StoreAddress = Builder.CreateConstInBoundsGEP2_64(
1771 Ty: ArrayType::get(ElementType: PtrTy, NumElements: NumCapturedVars), Ptr: Args, Idx0: 0, Idx1: Idx);
1772 Builder.CreateStore(Val: V, Ptr: StoreAddress);
1773 }
1774
1775 Value *Cond =
1776 IfCondition ? Builder.CreateSExtOrTrunc(V: IfCondition, DestTy: OMPIRBuilder->Int32)
1777 : Builder.getInt32(C: 1);
1778 Value *NumThreadsArg =
1779 NumThreads ? Builder.CreateZExtOrTrunc(V: NumThreads, DestTy: OMPIRBuilder->Int32)
1780 : Builder.getInt32(C: -1);
1781
1782 // If this is not a Generic kernel, we can skip generating the wrapper.
1783 Value *WrapperFn;
1784 if (isGenericKernel(Fn&: *OuterFn))
1785 WrapperFn = createTargetParallelWrapper(OMPIRBuilder, OutlinedFn);
1786 else
1787 WrapperFn = Constant::getNullValue(Ty: PtrTy);
1788
1789 // Build kmpc_parallel_60 call
1790 Value *Parallel60CallArgs[] = {
1791 /* identifier*/ Ident,
1792 /* global thread num*/ ThreadID,
1793 /* if expression */ Cond,
1794 /* number of threads */ NumThreadsArg,
1795 /* Proc bind */ Builder.getInt32(C: -1),
1796 /* outlined function */ &OutlinedFn,
1797 /* wrapper function */ WrapperFn,
1798 /* arguments of the outlined funciton*/ Args,
1799 /* number of arguments */ Builder.getInt64(C: NumCapturedVars),
1800 /* strict for number of threads */ Builder.getInt32(C: 0)};
1801
1802 FunctionCallee RTLFn =
1803 OMPIRBuilder->getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_parallel_60);
1804
1805 OMPIRBuilder->createRuntimeFunctionCall(Callee: RTLFn, Args: Parallel60CallArgs);
1806
1807 LLVM_DEBUG(dbgs() << "With kmpc_parallel_60 placed: "
1808 << *Builder.GetInsertBlock()->getParent() << "\n");
1809
1810 // Initialize the local TID stack location with the argument value.
1811 Builder.SetInsertPoint(PrivTID);
1812 Function::arg_iterator OutlinedAI = OutlinedFn.arg_begin();
1813 Builder.CreateStore(Val: Builder.CreateLoad(Ty: OMPIRBuilder->Int32, Ptr: OutlinedAI),
1814 Ptr: PrivTIDAddr);
1815
1816 // Remove redundant call to the outlined function.
1817 CI->eraseFromParent();
1818
1819 for (Instruction *I : ToBeDeleted) {
1820 I->eraseFromParent();
1821 }
1822}
1823
1824// Callback used to create OpenMP runtime calls to support
1825// omp parallel clause for the host.
1826// We need to use this callback to replace call to the OutlinedFn in OuterFn
1827// by the call to the OpenMP host runtime function ( __kmpc_fork_call[_if])
1828static void
1829hostParallelCallback(OpenMPIRBuilder *OMPIRBuilder, Function &OutlinedFn,
1830 Function *OuterFn, Value *Ident, Value *IfCondition,
1831 Instruction *PrivTID, AllocaInst *PrivTIDAddr,
1832 const SmallVector<Instruction *, 4> &ToBeDeleted) {
1833 IRBuilder<> &Builder = OMPIRBuilder->Builder;
1834 FunctionCallee RTLFn;
1835 if (IfCondition) {
1836 RTLFn =
1837 OMPIRBuilder->getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_fork_call_if);
1838 } else {
1839 RTLFn =
1840 OMPIRBuilder->getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_fork_call);
1841 }
1842 if (auto *F = dyn_cast<Function>(Val: RTLFn.getCallee())) {
1843 if (!F->hasMetadata(KindID: LLVMContext::MD_callback)) {
1844 LLVMContext &Ctx = F->getContext();
1845 MDBuilder MDB(Ctx);
1846 // Annotate the callback behavior of the __kmpc_fork_call:
1847 // - The callback callee is argument number 2 (microtask).
1848 // - The first two arguments of the callback callee are unknown (-1).
1849 // - All variadic arguments to the __kmpc_fork_call are passed to the
1850 // callback callee.
1851 F->addMetadata(KindID: LLVMContext::MD_callback,
1852 MD&: *MDNode::get(Context&: Ctx, MDs: {MDB.createCallbackEncoding(
1853 CalleeArgNo: 2, Arguments: {-1, -1},
1854 /* VarArgsArePassed */ true)}));
1855 }
1856 }
1857 // Add some known attributes.
1858 OutlinedFn.addParamAttr(ArgNo: 0, Kind: Attribute::NoAlias);
1859 OutlinedFn.addParamAttr(ArgNo: 1, Kind: Attribute::NoAlias);
1860 OutlinedFn.addFnAttr(Kind: Attribute::NoUnwind);
1861
1862 assert(OutlinedFn.arg_size() >= 2 &&
1863 "Expected at least tid and bounded tid as arguments");
1864 unsigned NumCapturedVars = OutlinedFn.arg_size() - /* tid & bounded tid */ 2;
1865
1866 CallInst *CI = cast<CallInst>(Val: OutlinedFn.user_back());
1867
1868 // CodeExtractor creates an alloca aggregate struct containing pointers to all
1869 // the captured values. The address of this struct is then passed as a
1870 // parameter to the outlined function. This parameter of the outlined function
1871 // can be marked noalias because CodeExractor + this host OpenMP lowering
1872 // maintains the following invariants:
1873 // - The caller initializes the fields before starting any callbacks.
1874 // - Neither the caller, OpenMP runtime, nor callbacks modify the structure
1875 // while callbacks are executing.
1876 // - The callbacks only load captured values from the structure; they do not
1877 // use it to write results back to the caller. This is generated code; the
1878 // user has no way to modify the structure directly.
1879 //
1880 // Multiple callbacks may read the same structure. This is allowed by noalias,
1881 // which permits aliasing accesses to memory that is not modified during the
1882 // call.
1883 //
1884 // The synchronous fork keeps the structure alive until all callbacks return,
1885 // which justifies nofreeobj. Neither attribute extends to the objects pointed
1886 // to by the captured fields.
1887 if (NumCapturedVars) {
1888 assert(NumCapturedVars == 1 && "Expected an aggregate capture argument");
1889 assert(isa<AllocaInst>(CI->getArgOperand(2)) &&
1890 cast<AllocaInst>(CI->getArgOperand(2))
1891 ->getAllocatedType()
1892 ->isStructTy() &&
1893 "Expected a stack-allocated capture structure");
1894
1895 OutlinedFn.addParamAttr(ArgNo: 2, Kind: Attribute::NoAlias);
1896 OutlinedFn.addParamAttr(ArgNo: 2, Kind: Attribute::NoFreeObj);
1897 }
1898
1899 CI->getParent()->setName("omp_parallel");
1900 Builder.SetInsertPoint(CI);
1901
1902 // Build call __kmpc_fork_call[_if](Ident, n, microtask, var1, .., varn);
1903 Value *ForkCallArgs[] = {Ident, Builder.getInt32(C: NumCapturedVars),
1904 &OutlinedFn};
1905
1906 SmallVector<Value *, 16> RealArgs;
1907 RealArgs.append(in_start: std::begin(arr&: ForkCallArgs), in_end: std::end(arr&: ForkCallArgs));
1908 if (IfCondition) {
1909 Value *Cond = Builder.CreateSExtOrTrunc(V: IfCondition, DestTy: OMPIRBuilder->Int32);
1910 RealArgs.push_back(Elt: Cond);
1911 }
1912 RealArgs.append(in_start: CI->arg_begin() + /* tid & bound tid */ 2, in_end: CI->arg_end());
1913
1914 // __kmpc_fork_call_if always expects a void ptr as the last argument
1915 // If there are no arguments, pass a null pointer.
1916 auto PtrTy = OMPIRBuilder->VoidPtr;
1917 if (IfCondition && NumCapturedVars == 0) {
1918 Value *NullPtrValue = Constant::getNullValue(Ty: PtrTy);
1919 RealArgs.push_back(Elt: NullPtrValue);
1920 }
1921
1922 OMPIRBuilder->createRuntimeFunctionCall(Callee: RTLFn, Args: RealArgs);
1923
1924 LLVM_DEBUG(dbgs() << "With fork_call placed: "
1925 << *Builder.GetInsertBlock()->getParent() << "\n");
1926
1927 // Initialize the local TID stack location with the argument value.
1928 Builder.SetInsertPoint(PrivTID);
1929 Function::arg_iterator OutlinedAI = OutlinedFn.arg_begin();
1930 Builder.CreateStore(Val: Builder.CreateLoad(Ty: OMPIRBuilder->Int32, Ptr: OutlinedAI),
1931 Ptr: PrivTIDAddr);
1932
1933 // Remove redundant call to the outlined function.
1934 CI->eraseFromParent();
1935
1936 for (Instruction *I : ToBeDeleted) {
1937 I->eraseFromParent();
1938 }
1939}
1940
1941OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createParallel(
1942 const LocationDescription &Loc, InsertPointTy OuterAllocIP,
1943 ArrayRef<BasicBlock *> OuterDeallocBlocks, BodyGenCallbackTy BodyGenCB,
1944 PrivatizeCallbackTy PrivCB, FinalizeCallbackTy FiniCB, Value *IfCondition,
1945 Value *NumThreads, omp::ProcBindKind ProcBind, bool IsCancellable) {
1946 assert(!isConflictIP(Loc.IP, OuterAllocIP) && "IPs must not be ambiguous");
1947
1948 if (!updateToLocation(Loc))
1949 return Loc.IP;
1950
1951 uint32_t SrcLocStrSize;
1952 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1953 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
1954 const bool NeedThreadID = NumThreads || Config.isTargetDevice() ||
1955 (ProcBind != OMP_PROC_BIND_default);
1956 Value *ThreadID = NeedThreadID ? getOrCreateThreadID(Ident) : nullptr;
1957 // If we generate code for the target device, we need to allocate
1958 // struct for aggregate params in the device default alloca address space.
1959 // OpenMP runtime requires that the params of the extracted functions are
1960 // passed as zero address space pointers. This flag ensures that extracted
1961 // function arguments are declared in zero address space
1962 bool ArgsInZeroAddressSpace = Config.isTargetDevice();
1963
1964 // Build call __kmpc_push_num_threads(&Ident, global_tid, num_threads)
1965 // only if we compile for host side.
1966 if (NumThreads && !Config.isTargetDevice()) {
1967 Value *Args[] = {
1968 Ident, ThreadID,
1969 Builder.CreateIntCast(V: NumThreads, DestTy: Int32, /*isSigned*/ false)};
1970 createRuntimeFunctionCall(
1971 Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_push_num_threads), Args);
1972 }
1973
1974 if (ProcBind != OMP_PROC_BIND_default) {
1975 // Build call __kmpc_push_proc_bind(&Ident, global_tid, proc_bind)
1976 Value *Args[] = {
1977 Ident, ThreadID,
1978 ConstantInt::get(Ty: Int32, V: unsigned(ProcBind), /*isSigned=*/IsSigned: true)};
1979 createRuntimeFunctionCall(
1980 Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_push_proc_bind), Args);
1981 }
1982
1983 BasicBlock *InsertBB = Builder.GetInsertBlock();
1984 Function *OuterFn = InsertBB->getParent();
1985
1986 // Save the outer alloca block because the insertion iterator may get
1987 // invalidated and we still need this later.
1988 BasicBlock *OuterAllocaBlock = OuterAllocIP.getNodeParent();
1989
1990 // Vector to remember instructions we used only during the modeling but which
1991 // we want to delete at the end.
1992 SmallVector<Instruction *, 4> ToBeDeleted;
1993
1994 // Change the location to the outer alloca insertion point to create and
1995 // initialize the allocas we pass into the parallel region.
1996 InsertPointTy NewOuter(OuterAllocaBlock->begin());
1997 Builder.restoreIP(IP: NewOuter);
1998 AllocaInst *TIDAddrAlloca = Builder.CreateAlloca(Ty: Int32, ArraySize: nullptr, Name: "tid.addr");
1999 AllocaInst *ZeroAddrAlloca =
2000 Builder.CreateAlloca(Ty: Int32, ArraySize: nullptr, Name: "zero.addr");
2001 Instruction *TIDAddr = TIDAddrAlloca;
2002 Instruction *ZeroAddr = ZeroAddrAlloca;
2003 if (ArgsInZeroAddressSpace && M.getDataLayout().getAllocaAddrSpace() != 0) {
2004 // Add additional casts to enforce pointers in zero address space
2005 TIDAddr = new AddrSpaceCastInst(
2006 TIDAddrAlloca, PointerType ::get(C&: M.getContext(), AddressSpace: 0), "tid.addr.ascast");
2007 TIDAddr->insertAfter(InsertPos: TIDAddrAlloca->getIterator());
2008 ToBeDeleted.push_back(Elt: TIDAddr);
2009 ZeroAddr = new AddrSpaceCastInst(ZeroAddrAlloca,
2010 PointerType ::get(C&: M.getContext(), AddressSpace: 0),
2011 "zero.addr.ascast");
2012 ZeroAddr->insertAfter(InsertPos: ZeroAddrAlloca->getIterator());
2013 ToBeDeleted.push_back(Elt: ZeroAddr);
2014 }
2015
2016 // We only need TIDAddr and ZeroAddr for modeling purposes to get the
2017 // associated arguments in the outlined function, so we delete them later.
2018 ToBeDeleted.push_back(Elt: TIDAddrAlloca);
2019 ToBeDeleted.push_back(Elt: ZeroAddrAlloca);
2020
2021 // Create an artificial insertion point that will also ensure the blocks we
2022 // are about to split are not degenerated.
2023 auto *UI = new UnreachableInst(Builder.getContext(), InsertBB);
2024 UI->setDebugLoc(Loc.DL);
2025
2026 BasicBlock *EntryBB = UI->getParent();
2027 BasicBlock *PRegEntryBB = EntryBB->splitBasicBlock(I: UI, BBName: "omp.par.entry");
2028 BasicBlock *PRegBodyBB = PRegEntryBB->splitBasicBlock(I: UI, BBName: "omp.par.region");
2029 BasicBlock *PRegPreFiniBB =
2030 PRegBodyBB->splitBasicBlock(I: UI, BBName: "omp.par.pre_finalize");
2031 BasicBlock *PRegExitBB = PRegPreFiniBB->splitBasicBlock(I: UI, BBName: "omp.par.exit");
2032
2033 auto FiniCBWrapper = [&](InsertPointTy IP) {
2034 // Hide "open-ended" blocks from the given FiniCB by setting the right jump
2035 // target to the region exit block.
2036 if (IP == IP.getNodeParent()->end()) {
2037 IRBuilder<>::InsertPointGuard IPG(Builder);
2038 Builder.restoreIP(IP);
2039 Instruction *I = Builder.CreateBr(Dest: PRegExitBB);
2040 IP = I->getIterator();
2041 }
2042 assert(IP.getNodeParent()->getTerminator()->getNumSuccessors() == 1 &&
2043 IP.getNodeParent()->getTerminator()->getSuccessor(0) == PRegExitBB &&
2044 "Unexpected insertion point for finalization call!");
2045 return FiniCB(IP);
2046 };
2047
2048 FinalizationStack.push_back(Elt: {FiniCBWrapper, OMPD_parallel, IsCancellable});
2049
2050 // Generate the privatization allocas in the block that will become the entry
2051 // of the outlined function.
2052 Builder.SetInsertPoint(PRegEntryBB->getTerminator());
2053 InsertPointTy InnerAllocaIP = Builder.saveIP();
2054
2055 AllocaInst *PrivTIDAddr =
2056 Builder.CreateAlloca(Ty: Int32, ArraySize: nullptr, Name: "tid.addr.local");
2057 Instruction *PrivTID = Builder.CreateLoad(Ty: Int32, Ptr: PrivTIDAddr, Name: "tid");
2058
2059 // Add some fake uses for OpenMP provided arguments.
2060 ToBeDeleted.push_back(Elt: Builder.CreateLoad(Ty: Int32, Ptr: TIDAddr, Name: "tid.addr.use"));
2061 Instruction *ZeroAddrUse =
2062 Builder.CreateLoad(Ty: Int32, Ptr: ZeroAddr, Name: "zero.addr.use");
2063 ToBeDeleted.push_back(Elt: ZeroAddrUse);
2064
2065 // EntryBB
2066 // |
2067 // V
2068 // PRegionEntryBB <- Privatization allocas are placed here.
2069 // |
2070 // V
2071 // PRegionBodyBB <- BodeGen is invoked here.
2072 // |
2073 // V
2074 // PRegPreFiniBB <- The block we will start finalization from.
2075 // |
2076 // V
2077 // PRegionExitBB <- A common exit to simplify block collection.
2078 //
2079
2080 LLVM_DEBUG(dbgs() << "Before body codegen: " << *OuterFn << "\n");
2081
2082 // Let the caller create the body.
2083 assert(BodyGenCB && "Expected body generation callback!");
2084 InsertPointTy CodeGenIP(PRegBodyBB->begin());
2085 if (Error Err = BodyGenCB(InnerAllocaIP, CodeGenIP, PRegExitBB))
2086 return Err;
2087
2088 LLVM_DEBUG(dbgs() << "After body codegen: " << *OuterFn << "\n");
2089
2090 // If OuterFn is a Generic kernel, we need to use device shared memory to
2091 // allocate argument structures. Otherwise, we use stack allocations as usual.
2092 bool UsesDeviceSharedMemory =
2093 Config.isTargetDevice() && isGenericKernel(Fn&: *OuterFn);
2094 std::unique_ptr<OutlineInfo> OI =
2095 UsesDeviceSharedMemory
2096 ? std::make_unique<DeviceSharedMemOutlineInfo>(args&: *this)
2097 : std::make_unique<OutlineInfo>();
2098
2099 if (Config.isTargetDevice()) {
2100 // Generate OpenMP target specific runtime call
2101 OI->PostOutlineCB = [=, ToBeDeletedVec =
2102 std::move(ToBeDeleted)](Function &OutlinedFn) {
2103 targetParallelCallback(OMPIRBuilder: this, OutlinedFn, OuterFn, OuterAllocaBB: OuterAllocaBlock, Ident,
2104 IfCondition, NumThreads, PrivTID, PrivTIDAddr,
2105 ThreadID, ToBeDeleted: ToBeDeletedVec);
2106 };
2107 } else {
2108 // Generate OpenMP host runtime call
2109 OI->PostOutlineCB = [=, ToBeDeletedVec =
2110 std::move(ToBeDeleted)](Function &OutlinedFn) {
2111 hostParallelCallback(OMPIRBuilder: this, OutlinedFn, OuterFn, Ident, IfCondition,
2112 PrivTID, PrivTIDAddr, ToBeDeleted: ToBeDeletedVec);
2113 };
2114 }
2115
2116 OI->FixUpNonEntryAllocas = true;
2117 OI->OuterAllocBB = OuterAllocaBlock;
2118 OI->EntryBB = PRegEntryBB;
2119 OI->ExitBB = PRegExitBB;
2120 OI->OuterDeallocBBs.reserve(N: OuterDeallocBlocks.size());
2121 copy(Range&: OuterDeallocBlocks, Out: OI->OuterDeallocBBs.end());
2122
2123 SmallPtrSet<BasicBlock *, 32> ParallelRegionBlockSet;
2124 SmallVector<BasicBlock *, 32> Blocks;
2125 OI->collectBlocks(BlockSet&: ParallelRegionBlockSet, BlockVector&: Blocks);
2126
2127 CodeExtractorAnalysisCache CEAC(*OuterFn);
2128 CodeExtractor Extractor(Blocks, /* DominatorTree */ nullptr,
2129 /* AggregateArgs */ false,
2130 /* BlockFrequencyInfo */ nullptr,
2131 /* BranchProbabilityInfo */ nullptr,
2132 /* AssumptionCache */ nullptr,
2133 /* AllowVarArgs */ true,
2134 /* AllowAlloca */ true,
2135 /* AllocationBlock */ OuterAllocaBlock,
2136 /* DeallocationBlocks */ {},
2137 /* Suffix */ ".omp_par", ArgsInZeroAddressSpace);
2138
2139 // Find inputs to, outputs from the code region.
2140 BasicBlock *CommonExit = nullptr;
2141 SetVector<Value *> Inputs, Outputs, SinkingCands, HoistingCands;
2142 Extractor.findAllocas(CEAC, SinkCands&: SinkingCands, HoistCands&: HoistingCands, ExitBlock&: CommonExit);
2143
2144 Extractor.findInputsOutputs(Inputs, Outputs, Allocas: SinkingCands,
2145 /*CollectGlobalInputs=*/true);
2146
2147 Inputs.remove_if(P: [&](Value *I) {
2148 if (auto *GV = dyn_cast_if_present<GlobalVariable>(Val: I))
2149 return GV->getValueType() == OpenMPIRBuilder::Ident;
2150
2151 return false;
2152 });
2153
2154 LLVM_DEBUG(dbgs() << "Before privatization: " << *OuterFn << "\n");
2155
2156 FunctionCallee TIDRTLFn =
2157 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_global_thread_num);
2158
2159 auto PrivHelper = [&](Value &V) -> Error {
2160 if (&V == TIDAddr || &V == ZeroAddr) {
2161 OI->ExcludeArgsFromAggregate.push_back(Elt: &V);
2162 return Error::success();
2163 }
2164
2165 SetVector<Use *> Uses;
2166 for (Use &U : V.uses())
2167 if (auto *UserI = dyn_cast<Instruction>(Val: U.getUser()))
2168 if (ParallelRegionBlockSet.count(Ptr: UserI->getParent()))
2169 Uses.insert(X: &U);
2170
2171 // __kmpc_fork_call expects extra arguments as pointers. If the input
2172 // already has a pointer type, everything is fine. Otherwise, store the
2173 // value onto stack and load it back inside the to-be-outlined region. This
2174 // will ensure only the pointer will be passed to the function.
2175 // FIXME: if there are more than 15 trailing arguments, they must be
2176 // additionally packed in a struct.
2177 Value *Inner = &V;
2178 if (!V.getType()->isPointerTy()) {
2179 IRBuilder<>::InsertPointGuard Guard(Builder);
2180 LLVM_DEBUG(llvm::dbgs() << "Forwarding input as pointer: " << V << "\n");
2181
2182 Builder.restoreIP(IP: OuterAllocIP);
2183 Value *Ptr;
2184 if (UsesDeviceSharedMemory) {
2185 // Use device shared memory instead, if needed.
2186 Ptr = createOMPAllocShared(Loc: Builder, VarType: V.getType(),
2187 Name: V.getName() + ".reloaded");
2188 for (BasicBlock *DeallocBlock : OuterDeallocBlocks) {
2189 assert(DeallocBlock->getParent() ==
2190 OuterAllocIP.getNodeParent()->getParent() &&
2191 "Dealloc block must be in the allocation's function to reuse "
2192 "its debug location");
2193 createOMPFreeShared(Loc: {DeallocBlock->getFirstInsertionPt(),
2194 Builder.getCurrentDebugLocation()},
2195 Addr: Ptr, VarType: V.getType());
2196 }
2197 } else {
2198 Ptr = Builder.CreateAlloca(Ty: V.getType(), ArraySize: nullptr,
2199 Name: V.getName() + ".reloaded");
2200 }
2201
2202 // Store to stack at end of the block that currently branches to the entry
2203 // block of the to-be-outlined region.
2204 Builder.SetInsertPoint(InsertBB->getTerminator()->getIterator());
2205 Builder.CreateStore(Val: &V, Ptr);
2206
2207 // Load back next to allocations in the to-be-outlined region.
2208 Builder.restoreIP(IP: InnerAllocaIP);
2209 Inner = Builder.CreateLoad(Ty: V.getType(), Ptr);
2210 }
2211
2212 Value *ReplacementValue = nullptr;
2213 CallInst *CI = dyn_cast<CallInst>(Val: &V);
2214 if (CI && CI->getCalledFunction() == TIDRTLFn.getCallee()) {
2215 ReplacementValue = PrivTID;
2216 } else {
2217 InsertPointOrErrorTy AfterIP =
2218 PrivCB(InnerAllocaIP, Builder.saveIP(), V, *Inner, ReplacementValue);
2219 if (!AfterIP)
2220 return AfterIP.takeError();
2221 Builder.restoreIP(IP: *AfterIP);
2222 InnerAllocaIP =
2223 InnerAllocaIP.getNodeParent()->getTerminator()->getIterator();
2224
2225 assert(ReplacementValue &&
2226 "Expected copy/create callback to set replacement value!");
2227 if (ReplacementValue == &V)
2228 return Error::success();
2229 }
2230
2231 for (Use *UPtr : Uses)
2232 UPtr->set(ReplacementValue);
2233
2234 return Error::success();
2235 };
2236
2237 // Reset the inner alloca insertion as it will be used for loading the values
2238 // wrapped into pointers before passing them into the to-be-outlined region.
2239 // Configure it to insert immediately after the fake use of zero address so
2240 // that they are available in the generated body and so that the
2241 // OpenMP-related values (thread ID and zero address pointers) remain leading
2242 // in the argument list.
2243 InnerAllocaIP = ZeroAddrUse->getNextNode()->getIterator();
2244
2245 // Reset the outer alloca insertion point to the entry of the relevant block
2246 // in case it was invalidated.
2247 OuterAllocIP = OuterAllocaBlock->getFirstInsertionPt();
2248
2249 for (Value *Input : Inputs) {
2250 LLVM_DEBUG(dbgs() << "Captured input: " << *Input << "\n");
2251 if (Error Err = PrivHelper(*Input))
2252 return Err;
2253 }
2254 LLVM_DEBUG({
2255 for (Value *Output : Outputs)
2256 LLVM_DEBUG(dbgs() << "Captured output: " << *Output << "\n");
2257 });
2258 assert(Outputs.empty() &&
2259 "OpenMP outlining should not produce live-out values!");
2260
2261 LLVM_DEBUG(dbgs() << "After privatization: " << *OuterFn << "\n");
2262 LLVM_DEBUG({
2263 for (auto *BB : Blocks)
2264 dbgs() << " PBR: " << BB->getName() << "\n";
2265 });
2266
2267 // Adjust the finalization stack, verify the adjustment, and call the
2268 // finalize function a last time to finalize values between the pre-fini
2269 // block and the exit block if we left the parallel "the normal way".
2270 auto FiniInfo = FinalizationStack.pop_back_val();
2271 (void)FiniInfo;
2272 assert(FiniInfo.DK == OMPD_parallel &&
2273 "Unexpected finalization stack state!");
2274
2275 Instruction *PRegPreFiniTI = PRegPreFiniBB->getTerminator();
2276
2277 InsertPointTy PreFiniIP(PRegPreFiniTI->getIterator());
2278 Expected<BasicBlock *> FiniBBOrErr = FiniInfo.getFiniBB(Builder);
2279 if (!FiniBBOrErr)
2280 return FiniBBOrErr.takeError();
2281 {
2282 IRBuilderBase::InsertPointGuard Guard(Builder);
2283 Builder.restoreIP(IP: PreFiniIP);
2284 Builder.CreateBr(Dest: *FiniBBOrErr);
2285 // There's currently a branch to omp.par.exit. Delete it. We will get there
2286 // via the fini block
2287 Builder.GetInsertBlock()->getTerminator()->eraseFromParent();
2288 }
2289
2290 // Register the outlined info.
2291 addOutlineInfo(OI: std::move(OI));
2292
2293 InsertPointTy AfterIP(UI->getParent()->end());
2294 UI->eraseFromParent();
2295
2296 return AfterIP;
2297}
2298
2299void OpenMPIRBuilder::emitFlush(const LocationDescription &Loc) {
2300 // Build call void __kmpc_flush(ident_t *loc)
2301 uint32_t SrcLocStrSize;
2302 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2303 Value *Args[] = {getOrCreateIdent(SrcLocStr, SrcLocStrSize)};
2304
2305 createRuntimeFunctionCall(Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_flush),
2306 Args);
2307}
2308
2309void OpenMPIRBuilder::createFlush(const LocationDescription &Loc) {
2310 if (!updateToLocation(Loc))
2311 return;
2312 emitFlush(Loc);
2313}
2314
2315void OpenMPIRBuilder::createError(const LocationDescription &Loc, bool IsFatal,
2316 Value *Message) {
2317 if (!updateToLocation(Loc))
2318 return;
2319
2320 // Build call void __kmpc_error(ident_t *loc, int severity,
2321 // const char *message)
2322 uint32_t SrcLocStrSize;
2323 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2324 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2325 // Severity: 1 = warning, 2 = fatal.
2326 Value *Severity = ConstantInt::get(Ty: Int32, V: IsFatal ? 2 : 1);
2327 Value *MessageArg = Message ? Message : ConstantPointerNull::get(T: Int8Ptr);
2328 Value *Args[] = {Ident, Severity, MessageArg};
2329
2330 createRuntimeFunctionCall(Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_error),
2331 Args);
2332}
2333
2334void OpenMPIRBuilder::emitTaskyieldImpl(const LocationDescription &Loc) {
2335 // Build call __kmpc_omp_taskyield(loc, thread_id, 0);
2336 uint32_t SrcLocStrSize;
2337 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2338 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2339 Constant *I32Null = ConstantInt::getNullValue(Ty: Int32);
2340 Value *Args[] = {Ident, getOrCreateThreadID(Ident), I32Null};
2341
2342 createRuntimeFunctionCall(
2343 Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_taskyield), Args);
2344}
2345
2346void OpenMPIRBuilder::createTaskyield(const LocationDescription &Loc) {
2347 if (!updateToLocation(Loc))
2348 return;
2349 emitTaskyieldImpl(Loc);
2350}
2351
2352void OpenMPIRBuilder::emitTaskDependency(IRBuilderBase &Builder, Value *Entry,
2353 const DependData &Dep) {
2354 // Store the pointer to the variable
2355 Value *Addr = Builder.CreateStructGEP(
2356 Ty: DependInfo, Ptr: Entry,
2357 Idx: static_cast<unsigned int>(RTLDependInfoFields::BaseAddr));
2358 Value *DepValPtr = Builder.CreatePtrToInt(V: Dep.DepVal, DestTy: SizeTy);
2359 Builder.CreateStore(Val: DepValPtr, Ptr: Addr);
2360 // Store the size of the variable
2361 Value *Size = Builder.CreateStructGEP(
2362 Ty: DependInfo, Ptr: Entry, Idx: static_cast<unsigned int>(RTLDependInfoFields::Len));
2363 Builder.CreateStore(
2364 Val: ConstantInt::get(Ty: SizeTy,
2365 V: M.getDataLayout().getTypeStoreSize(Ty: Dep.DepValueType)),
2366 Ptr: Size);
2367 // Store the dependency kind
2368 Value *Flags = Builder.CreateStructGEP(
2369 Ty: DependInfo, Ptr: Entry, Idx: static_cast<unsigned int>(RTLDependInfoFields::Flags));
2370 Builder.CreateStore(Val: ConstantInt::get(Ty: Builder.getInt8Ty(),
2371 V: static_cast<unsigned int>(Dep.DepKind)),
2372 Ptr: Flags);
2373}
2374
2375// Processes the dependencies in Dependencies and does the following
2376// - Allocates space on the stack of an array of DependInfo objects
2377// - Populates each DependInfo object with relevant information of
2378// the corresponding dependence.
2379// - All code is inserted in the entry block of the current function.
2380static Value *emitTaskDependencies(
2381 OpenMPIRBuilder &OMPBuilder,
2382 const SmallVectorImpl<OpenMPIRBuilder::DependData> &Dependencies) {
2383 // Early return if we have no dependencies to process
2384 if (Dependencies.empty())
2385 return nullptr;
2386
2387 // Given a vector of DependData objects, in this function we create an
2388 // array on the stack that holds kmp_depend_info objects corresponding
2389 // to each dependency. This is then passed to the OpenMP runtime.
2390 // For example, if there are 'n' dependencies then the following psedo
2391 // code is generated. Assume the first dependence is on a variable 'a'
2392 //
2393 // \code{c}
2394 // DepArray = alloc(n x sizeof(kmp_depend_info);
2395 // idx = 0;
2396 // DepArray[idx].base_addr = ptrtoint(&a);
2397 // DepArray[idx].len = 8;
2398 // DepArray[idx].flags = Dep.DepKind; /*(See OMPContants.h for DepKind)*/
2399 // ++idx;
2400 // DepArray[idx].base_addr = ...;
2401 // \endcode
2402
2403 IRBuilderBase &Builder = OMPBuilder.Builder;
2404 Type *DependInfo = OMPBuilder.DependInfo;
2405
2406 Value *DepArray = nullptr;
2407 Type *DepArrayTy = ArrayType::get(ElementType: DependInfo, NumElements: Dependencies.size());
2408 {
2409 // Use a InsertPointGuard to restore the location back along with the
2410 // insertion point.
2411 IRBuilderBase::InsertPointGuard IPGuard(Builder);
2412 Builder.SetInsertPoint(
2413 Builder.GetInsertBlock()->getParent()->getEntryBlock().getTerminator());
2414 DepArray = Builder.CreateAlloca(Ty: DepArrayTy, ArraySize: nullptr, Name: ".dep.arr.addr");
2415 }
2416
2417 for (const auto &[DepIdx, Dep] : enumerate(First: Dependencies)) {
2418 Value *Base =
2419 Builder.CreateConstInBoundsGEP2_64(Ty: DepArrayTy, Ptr: DepArray, Idx0: 0, Idx1: DepIdx);
2420 OMPBuilder.emitTaskDependency(Builder, Entry: Base, Dep);
2421 }
2422 return DepArray;
2423}
2424
2425void OpenMPIRBuilder::emitTaskwaitImpl(const LocationDescription &Loc) {
2426 // Build call kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
2427 // global_tid);
2428 uint32_t SrcLocStrSize;
2429 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2430 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2431 Value *Args[] = {Ident, getOrCreateThreadID(Ident)};
2432
2433 // Ignore return result until untied tasks are supported.
2434 createRuntimeFunctionCall(
2435 Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_taskwait), Args);
2436}
2437
2438void OpenMPIRBuilder::createTaskwait(const LocationDescription &Loc,
2439 DependenciesInfo Dependencies,
2440 bool IsNowait) {
2441 if (!updateToLocation(Loc))
2442 return;
2443
2444 Value *DepArray = nullptr;
2445 Type *DepArrayTy = nullptr;
2446 Value *NumDeps = nullptr;
2447 if (Dependencies.DepArray) {
2448 DepArray = Dependencies.DepArray;
2449 NumDeps = Dependencies.NumDeps;
2450 } else if (!Dependencies.Deps.empty()) {
2451 DepArrayTy = ArrayType::get(ElementType: DependInfo, NumElements: Dependencies.Deps.size());
2452 NumDeps = Builder.getInt32(C: Dependencies.Deps.size());
2453 {
2454 IRBuilderBase::InsertPointGuard IPGuard(Builder);
2455 BasicBlock &entryBB =
2456 Builder.GetInsertBlock()->getParent()->getEntryBlock();
2457 Builder.SetInsertPoint(entryBB.getFirstInsertionPt());
2458 DepArray = Builder.CreateAlloca(Ty: DepArrayTy, ArraySize: nullptr, Name: ".dep.arr.addr");
2459 }
2460
2461 for (const auto &[DepIdx, Dep] : enumerate(First&: Dependencies.Deps)) {
2462 Value *Base =
2463 Builder.CreateConstInBoundsGEP2_64(Ty: DepArrayTy, Ptr: DepArray, Idx0: 0, Idx1: DepIdx);
2464 this->emitTaskDependency(Builder, Entry: Base, Dep);
2465 }
2466 }
2467
2468 if (DepArray) {
2469 uint32_t SrcLocStrSize;
2470 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2471 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2472 Value *Args[] = {
2473 Ident,
2474 getOrCreateThreadID(Ident),
2475 NumDeps,
2476 DepArray,
2477 ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0),
2478 ConstantPointerNull::get(T: PointerType::getUnqual(C&: M.getContext())),
2479 ConstantInt::get(Ty: Builder.getInt32Ty(), V: IsNowait)};
2480 createRuntimeFunctionCall(
2481 Callee: getOrCreateRuntimeFunctionPtr(
2482 FnID: omp::RuntimeFunction::OMPRTL___kmpc_omp_taskwait_deps_51),
2483 Args);
2484 } else {
2485 emitTaskwaitImpl(Loc);
2486 }
2487}
2488
2489/// Create the task duplication function passed to kmpc_taskloop.
2490Expected<Value *> OpenMPIRBuilder::createTaskDuplicationFunction(
2491 Type *PrivatesTy, int32_t PrivatesIndex, TaskDupCallbackTy DupCB) {
2492 unsigned ProgramAddressSpace = M.getDataLayout().getProgramAddressSpace();
2493 if (!DupCB)
2494 return Constant::getNullValue(
2495 Ty: PointerType::get(C&: Builder.getContext(), AddressSpace: ProgramAddressSpace));
2496
2497 // From OpenMP Runtime p_task_dup_t:
2498 // Routine optionally generated by the compiler for setting the lastprivate
2499 // flag and calling needed constructors for private/firstprivate objects (used
2500 // to form taskloop tasks from pattern task) Parameters: dest task, src task,
2501 // lastprivate flag.
2502 // typedef void (*p_task_dup_t)(kmp_task_t *, kmp_task_t *, kmp_int32);
2503
2504 auto *VoidPtrTy = PointerType::get(C&: Builder.getContext(), AddressSpace: ProgramAddressSpace);
2505
2506 FunctionType *DupFuncTy = FunctionType::get(
2507 Result: Builder.getVoidTy(), Params: {VoidPtrTy, VoidPtrTy, Builder.getInt32Ty()},
2508 /*isVarArg=*/false);
2509
2510 Function *DupFunction = Function::Create(Ty: DupFuncTy, Linkage: Function::InternalLinkage,
2511 N: "omp_taskloop_dup", M);
2512 Value *DestTaskArg = DupFunction->getArg(i: 0);
2513 Value *SrcTaskArg = DupFunction->getArg(i: 1);
2514 Value *LastprivateFlagArg = DupFunction->getArg(i: 2);
2515 DestTaskArg->setName("dest_task");
2516 SrcTaskArg->setName("src_task");
2517 LastprivateFlagArg->setName("lastprivate_flag");
2518
2519 IRBuilderBase::InsertPointGuard Guard(Builder);
2520 Builder.SetInsertPoint(
2521 BasicBlock::Create(Context&: Builder.getContext(), Name: "entry", Parent: DupFunction));
2522
2523 auto GetTaskContextPtrFromArg = [&](Value *Arg) -> Value * {
2524 Type *TaskWithPrivatesTy =
2525 StructType::get(Context&: Builder.getContext(), Elements: {Task, PrivatesTy});
2526 Value *TaskPrivates = Builder.CreateGEP(
2527 Ty: TaskWithPrivatesTy, Ptr: Arg, IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: 1)});
2528 Value *ContextPtr = Builder.CreateGEP(
2529 Ty: PrivatesTy, Ptr: TaskPrivates,
2530 IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: PrivatesIndex)});
2531 return ContextPtr;
2532 };
2533
2534 Value *DestTaskContextPtr = GetTaskContextPtrFromArg(DestTaskArg);
2535 Value *SrcTaskContextPtr = GetTaskContextPtrFromArg(SrcTaskArg);
2536
2537 DestTaskContextPtr->setName("destPtr");
2538 SrcTaskContextPtr->setName("srcPtr");
2539
2540 InsertPointTy AllocaIP(DupFunction->getEntryBlock().begin());
2541 InsertPointTy CodeGenIP = Builder.saveIP();
2542 Expected<IRBuilderBase::InsertPoint> AfterIPOrError =
2543 DupCB(AllocaIP, CodeGenIP, DestTaskContextPtr, SrcTaskContextPtr);
2544 if (!AfterIPOrError)
2545 return AfterIPOrError.takeError();
2546 Builder.restoreIP(IP: *AfterIPOrError);
2547
2548 Builder.CreateRetVoid();
2549
2550 return DupFunction;
2551}
2552
2553OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTaskloop(
2554 const LocationDescription &Loc, InsertPointTy AllocaIP,
2555 ArrayRef<BasicBlock *> DeallocBlocks, BodyGenCallbackTy BodyGenCB,
2556 llvm::function_ref<llvm::Expected<llvm::CanonicalLoopInfo *>()> LoopInfo,
2557 Value *LBVal, Value *UBVal, Value *StepVal, bool Untied, Value *IfCond,
2558 Value *GrainSize, bool NoGroup, int Sched, Value *Final, bool Mergeable,
2559 Value *Priority, uint64_t NumOfCollapseLoops, TaskDupCallbackTy DupCB,
2560 Value *TaskContextStructPtrVal, bool FreeAgent) {
2561
2562 if (!updateToLocation(Loc))
2563 return InsertPointTy();
2564
2565 uint32_t SrcLocStrSize;
2566 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2567 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2568
2569 BasicBlock *TaskloopExitBB =
2570 splitBB(Builder, /*CreateBranch=*/true, Name: "taskloop.exit");
2571 BasicBlock *TaskloopBodyBB =
2572 splitBB(Builder, /*CreateBranch=*/true, Name: "taskloop.body");
2573 BasicBlock *TaskloopAllocaBB =
2574 splitBB(Builder, /*CreateBranch=*/true, Name: "taskloop.alloca");
2575
2576 InsertPointTy TaskloopAllocaIP = TaskloopAllocaBB->begin();
2577 InsertPointTy TaskloopBodyIP = TaskloopBodyBB->begin();
2578
2579 if (Error Err = BodyGenCB(TaskloopAllocaIP, TaskloopBodyIP, TaskloopExitBB))
2580 return Err;
2581
2582 llvm::Expected<llvm::CanonicalLoopInfo *> result = LoopInfo();
2583 if (!result) {
2584 return result.takeError();
2585 }
2586
2587 llvm::CanonicalLoopInfo *CLI = result.get();
2588 if (CLI->VirtualCollapse)
2589 CLI->reportUnsupportedVirtualCollapse();
2590 auto OI = std::make_unique<OutlineInfo>();
2591 OI->EntryBB = TaskloopAllocaBB;
2592 OI->OuterAllocBB = AllocaIP.getNodeParent();
2593 OI->ExitBB = TaskloopExitBB;
2594 OI->OuterDeallocBBs.reserve(N: DeallocBlocks.size());
2595 copy(Range&: DeallocBlocks, Out: OI->OuterDeallocBBs.end());
2596
2597 // Add the thread ID argument.
2598 SmallVector<Instruction *> ToBeDeleted;
2599 // dummy instruction to be used as a fake argument
2600 OI->ExcludeArgsFromAggregate.push_back(Elt: createFakeIntVal(
2601 Builder, OuterAllocaIP: AllocaIP, ToBeDeleted, InnerAllocaIP: TaskloopAllocaIP, Name: "global.tid", AsPtr: false));
2602 Value *FakeLB = createFakeIntVal(Builder, OuterAllocaIP: AllocaIP, ToBeDeleted,
2603 InnerAllocaIP: TaskloopAllocaIP, Name: "lb", AsPtr: false, Is64Bit: true);
2604 Value *FakeUB = createFakeIntVal(Builder, OuterAllocaIP: AllocaIP, ToBeDeleted,
2605 InnerAllocaIP: TaskloopAllocaIP, Name: "ub", AsPtr: false, Is64Bit: true);
2606 Value *FakeStep = createFakeIntVal(Builder, OuterAllocaIP: AllocaIP, ToBeDeleted,
2607 InnerAllocaIP: TaskloopAllocaIP, Name: "step", AsPtr: false, Is64Bit: true);
2608 // For Taskloop, we want to force the bounds being the first 3 inputs in the
2609 // aggregate struct
2610 OI->Inputs.insert(X: FakeLB);
2611 OI->Inputs.insert(X: FakeUB);
2612 OI->Inputs.insert(X: FakeStep);
2613 if (TaskContextStructPtrVal)
2614 OI->Inputs.insert(X: TaskContextStructPtrVal);
2615 assert(((TaskContextStructPtrVal && DupCB) ||
2616 (!TaskContextStructPtrVal && !DupCB)) &&
2617 "Task context struct ptr and duplication callback must be both set "
2618 "or both null");
2619
2620 // It isn't safe to run the duplication bodygen callback inside the post
2621 // outlining callback so this has to be run now before we know the real task
2622 // shareds structure type.
2623 unsigned ProgramAddressSpace = M.getDataLayout().getProgramAddressSpace();
2624 Type *PointerTy = PointerType::get(C&: Builder.getContext(), AddressSpace: ProgramAddressSpace);
2625 Type *FakeSharedsTy = StructType::get(
2626 Context&: Builder.getContext(),
2627 Elements: {FakeLB->getType(), FakeUB->getType(), FakeStep->getType(), PointerTy});
2628 Expected<Value *> TaskDupFnOrErr = createTaskDuplicationFunction(
2629 PrivatesTy: FakeSharedsTy,
2630 /*PrivatesIndex: the pointer after the three indices above*/ PrivatesIndex: 3, DupCB);
2631 if (!TaskDupFnOrErr) {
2632 return TaskDupFnOrErr.takeError();
2633 }
2634 Value *TaskDupFn = *TaskDupFnOrErr;
2635
2636 OI->PostOutlineCB = [this, Ident, LBVal, UBVal, StepVal, Untied,
2637 TaskloopAllocaBB, CLI, TaskDupFn, ToBeDeleted, IfCond,
2638 GrainSize, NoGroup, Sched, FakeLB, FakeUB, FakeStep,
2639 FakeSharedsTy, Final, Mergeable, Priority,
2640 NumOfCollapseLoops,
2641 FreeAgent](Function &OutlinedFn) mutable {
2642 // Replace the Stale CI by appropriate RTL function call.
2643 assert(OutlinedFn.hasOneUse() &&
2644 "there must be a single user for the outlined function");
2645 CallInst *StaleCI = cast<CallInst>(Val: OutlinedFn.user_back());
2646
2647 /* Create the casting for the Bounds Values that can be used when outlining
2648 * to replace the uses of the fakes with real values */
2649 BasicBlock *CodeReplBB = StaleCI->getParent();
2650 Builder.SetInsertPoint(CodeReplBB->getFirstInsertionPt());
2651 Value *CastedLBVal =
2652 Builder.CreateIntCast(V: LBVal, DestTy: Builder.getInt64Ty(), isSigned: true, Name: "lb64");
2653 Value *CastedUBVal =
2654 Builder.CreateIntCast(V: UBVal, DestTy: Builder.getInt64Ty(), isSigned: true, Name: "ub64");
2655 Value *CastedStepVal =
2656 Builder.CreateIntCast(V: StepVal, DestTy: Builder.getInt64Ty(), isSigned: true, Name: "step64");
2657
2658 Builder.SetInsertPoint(StaleCI);
2659
2660 // Gather the arguments for emitting the runtime call for
2661 // @__kmpc_omp_task_alloc
2662 Function *TaskAllocFn =
2663 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_alloc);
2664
2665 Value *ThreadID = getOrCreateThreadID(Ident);
2666
2667 if (!NoGroup) {
2668 // Emit runtime call for @__kmpc_taskgroup
2669 Function *TaskgroupFn =
2670 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_taskgroup);
2671 Builder.CreateCall(Callee: TaskgroupFn, Args: {Ident, ThreadID});
2672 }
2673
2674 // `flags` Argument Configuration
2675 // Task is tied if (Flags & 1) == 1.
2676 // Task is untied if (Flags & 1) == 0.
2677 // Task is final if (Flags & 2) == 2.
2678 // Task is not final if (Flags & 2) == 0.
2679 // Task is mergeable if (Flags & 4) == 4.
2680 // Task is not mergeable if (Flags & 4) == 0.
2681 // Task is priority if (Flags & 32) == 32.
2682 // Task is not priority if (Flags & 32) == 0.
2683 // Task is free-agent eligible if (Flags & 128) == 128.
2684 // Task is not free-agent eligible if (Flags & 128) == 0.
2685 Value *Flags = Builder.getInt32(C: Untied ? 0 : 1);
2686 if (Final)
2687 Flags = Builder.CreateOr(LHS: Builder.getInt32(C: 2), RHS: Flags);
2688 if (Mergeable)
2689 Flags = Builder.CreateOr(LHS: Builder.getInt32(C: 4), RHS: Flags);
2690 if (Priority)
2691 Flags = Builder.CreateOr(LHS: Builder.getInt32(C: 32), RHS: Flags);
2692 if (FreeAgent)
2693 Flags = Builder.CreateOr(LHS: Builder.getInt32(C: 128), RHS: Flags);
2694
2695 Value *TaskSize = Builder.getInt64(
2696 C: divideCeil(Numerator: M.getDataLayout().getTypeSizeInBits(Ty: Task), Denominator: 8));
2697
2698 AllocaInst *ArgStructAlloca =
2699 dyn_cast<AllocaInst>(Val: StaleCI->getArgOperand(i: 1));
2700 assert(ArgStructAlloca &&
2701 "Unable to find the alloca instruction corresponding to arguments "
2702 "for extracted function");
2703 std::optional<TypeSize> ArgAllocSize =
2704 ArgStructAlloca->getAllocationSize(DL: M.getDataLayout());
2705 assert(ArgAllocSize &&
2706 "Unable to determine size of arguments for extracted function");
2707 Value *SharedsSize = Builder.getInt64(C: ArgAllocSize->getFixedValue());
2708
2709 // Emit the @__kmpc_omp_task_alloc runtime call
2710 // The runtime call returns a pointer to an area where the task captured
2711 // variables must be copied before the task is run (TaskData)
2712 CallInst *TaskData = Builder.CreateCall(
2713 Callee: TaskAllocFn, Args: {/*loc_ref=*/Ident, /*gtid=*/ThreadID, /*flags=*/Flags,
2714 /*sizeof_task=*/TaskSize, /*sizeof_shared=*/SharedsSize,
2715 /*task_func=*/&OutlinedFn});
2716
2717 Value *Shareds = StaleCI->getArgOperand(i: 1);
2718 Align Alignment = TaskData->getPointerAlignment(DL: M.getDataLayout());
2719 Value *TaskShareds = Builder.CreateLoad(Ty: VoidPtr, Ptr: TaskData);
2720 Builder.CreateMemCpy(Dst: TaskShareds, DstAlign: Alignment, Src: Shareds, SrcAlign: Alignment,
2721 Size: SharedsSize);
2722 // Get the pointer to loop lb, ub, step from task ptr
2723 // and set up the lowerbound,upperbound and step values
2724 llvm::Value *Lb = Builder.CreateGEP(
2725 Ty: FakeSharedsTy, Ptr: TaskShareds, IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: 0)});
2726
2727 llvm::Value *Ub = Builder.CreateGEP(
2728 Ty: FakeSharedsTy, Ptr: TaskShareds, IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: 1)});
2729
2730 llvm::Value *Step = Builder.CreateGEP(
2731 Ty: FakeSharedsTy, Ptr: TaskShareds, IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: 2)});
2732 llvm::Value *Loadstep = Builder.CreateLoad(Ty: Builder.getInt64Ty(), Ptr: Step);
2733
2734 // set up the arguments for emitting kmpc_taskloop runtime call
2735 // setting values for ifval, nogroup, sched, grainsize, task_dup
2736 Value *IfCondVal =
2737 IfCond ? Builder.CreateIntCast(V: IfCond, DestTy: Builder.getInt32Ty(), isSigned: true)
2738 : Builder.getInt32(C: 1);
2739 // As __kmpc_taskgroup is called manually in OMPIRBuilder, NoGroupVal should
2740 // always be 1 when calling __kmpc_taskloop to ensure it is not called again
2741 Value *NoGroupVal = Builder.getInt32(C: 1);
2742 Value *SchedVal = Builder.getInt32(C: Sched);
2743 Value *GrainSizeVal =
2744 GrainSize ? Builder.CreateIntCast(V: GrainSize, DestTy: Builder.getInt64Ty(), isSigned: true)
2745 : Builder.getInt64(C: 0);
2746 Value *TaskDup = TaskDupFn;
2747
2748 Value *Args[] = {Ident, ThreadID, TaskData, IfCondVal, Lb, Ub,
2749 Loadstep, NoGroupVal, SchedVal, GrainSizeVal, TaskDup};
2750
2751 // taskloop runtime call
2752 Function *TaskloopFn =
2753 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_taskloop);
2754 Builder.CreateCall(Callee: TaskloopFn, Args);
2755
2756 // Emit the @__kmpc_end_taskgroup runtime call to end the taskgroup if
2757 // nogroup is not defined
2758 if (!NoGroup) {
2759 Function *EndTaskgroupFn =
2760 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_end_taskgroup);
2761 Builder.CreateCall(Callee: EndTaskgroupFn, Args: {Ident, ThreadID});
2762 }
2763
2764 StaleCI->eraseFromParent();
2765
2766 Builder.SetInsertPoint(TaskloopAllocaBB->begin());
2767
2768 LoadInst *SharedsOutlined =
2769 Builder.CreateLoad(Ty: VoidPtr, Ptr: OutlinedFn.getArg(i: 1));
2770 OutlinedFn.getArg(i: 1)->replaceUsesWithIf(
2771 New: SharedsOutlined,
2772 ShouldReplace: [SharedsOutlined](Use &U) { return U.getUser() != SharedsOutlined; });
2773
2774 Value *IV = CLI->getIndVar();
2775 Type *IVTy = IV->getType();
2776 Constant *One = ConstantInt::get(Ty: Builder.getInt64Ty(), V: 1);
2777
2778 // When outlining, CodeExtractor will create GEP's to the LowerBound and
2779 // UpperBound. These GEP's can be reused for loading the tasks respective
2780 // bounds.
2781 Value *TaskLB = nullptr;
2782 Value *TaskUB = nullptr;
2783 Value *TaskStep = nullptr;
2784 Value *LoadTaskLB = nullptr;
2785 Value *LoadTaskUB = nullptr;
2786 Value *LoadTaskStep = nullptr;
2787 for (Instruction &I : *TaskloopAllocaBB) {
2788 if (I.getOpcode() == Instruction::GetElementPtr) {
2789 GetElementPtrInst &Gep = cast<GetElementPtrInst>(Val&: I);
2790 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: Gep.getOperand(i_nocapture: 2))) {
2791 switch (CI->getZExtValue()) {
2792 case 0:
2793 TaskLB = &I;
2794 break;
2795 case 1:
2796 TaskUB = &I;
2797 break;
2798 case 2:
2799 TaskStep = &I;
2800 break;
2801 }
2802 }
2803 } else if (I.getOpcode() == Instruction::Load) {
2804 LoadInst &Load = cast<LoadInst>(Val&: I);
2805 if (Load.getPointerOperand() == TaskLB) {
2806 assert(TaskLB != nullptr && "Expected value for TaskLB");
2807 LoadTaskLB = &I;
2808 } else if (Load.getPointerOperand() == TaskUB) {
2809 assert(TaskUB != nullptr && "Expected value for TaskUB");
2810 LoadTaskUB = &I;
2811 } else if (Load.getPointerOperand() == TaskStep) {
2812 assert(TaskStep != nullptr && "Expected value for TaskStep");
2813 LoadTaskStep = &I;
2814 }
2815 }
2816 }
2817
2818 Builder.SetInsertPoint(CLI->getPreheader()->getTerminator());
2819
2820 assert(LoadTaskLB != nullptr && "Expected value for LoadTaskLB");
2821 assert(LoadTaskUB != nullptr && "Expected value for LoadTaskUB");
2822 assert(LoadTaskStep != nullptr && "Expected value for LoadTaskStep");
2823 Value *TripCountMinusOne = Builder.CreateSDiv(
2824 LHS: Builder.CreateSub(LHS: LoadTaskUB, RHS: LoadTaskLB), RHS: LoadTaskStep);
2825 Value *TripCount = Builder.CreateAdd(LHS: TripCountMinusOne, RHS: One, Name: "trip_cnt");
2826 Value *CastedTripCount = Builder.CreateIntCast(V: TripCount, DestTy: IVTy, isSigned: true);
2827 Value *CastedTaskLB = Builder.CreateIntCast(V: LoadTaskLB, DestTy: IVTy, isSigned: true);
2828 // set the trip count in the CLI
2829 CLI->setTripCount(CastedTripCount);
2830
2831 Builder.SetInsertPoint(CLI->getBody()->getFirstInsertionPt());
2832
2833 if (NumOfCollapseLoops > 1) {
2834 llvm::SmallVector<User *> UsersToReplace;
2835 // When using the collapse clause, the bounds of the loop have to be
2836 // adjusted to properly represent the iterator of the outer loop.
2837 Value *IVPlusTaskLB = Builder.CreateAdd(
2838 LHS: CLI->getIndVar(),
2839 RHS: Builder.CreateSub(LHS: CastedTaskLB, RHS: ConstantInt::get(Ty: IVTy, V: 1)));
2840 // To ensure every Use is correctly captured, we first want to record
2841 // which users to replace the value in, and then replace the value.
2842 for (auto IVUse = CLI->getIndVar()->uses().begin();
2843 IVUse != CLI->getIndVar()->uses().end(); IVUse++) {
2844 User *IVUser = IVUse->getUser();
2845 if (auto *Op = dyn_cast<BinaryOperator>(Val: IVUser)) {
2846 if (Op->getOpcode() == Instruction::URem ||
2847 Op->getOpcode() == Instruction::UDiv) {
2848 UsersToReplace.push_back(Elt: IVUser);
2849 }
2850 }
2851 }
2852 for (User *User : UsersToReplace) {
2853 User->replaceUsesOfWith(From: CLI->getIndVar(), To: IVPlusTaskLB);
2854 }
2855 } else {
2856 // The canonical loop is generated with a fixed lower bound. We need to
2857 // update the index calculation code to use the task's lower bound. The
2858 // generated code looks like this:
2859 // %omp_loop.iv = phi ...
2860 // ...
2861 // %tmp = mul [type] %omp_loop.iv, step
2862 // %user_index = add [type] tmp, lb
2863 // OpenMPIRBuilder constructs canonical loops to have exactly three uses
2864 // of the normalised induction variable:
2865 // 1. This one: converting the normalised IV to the user IV
2866 // 2. The increment (add)
2867 // 3. The comparison against the trip count (icmp)
2868 // (1) is the only use that is a mul followed by an add so this cannot
2869 // match other IR.
2870 assert(CLI->getIndVar()->getNumUses() == 3 &&
2871 "Canonical loop should have exactly three uses of the ind var");
2872 for (User *IVUser : CLI->getIndVar()->users()) {
2873 if (auto *Mul = dyn_cast<BinaryOperator>(Val: IVUser)) {
2874 if (Mul->getOpcode() == Instruction::Mul) {
2875 for (User *MulUser : Mul->users()) {
2876 if (auto *Add = dyn_cast<BinaryOperator>(Val: MulUser)) {
2877 if (Add->getOpcode() == Instruction::Add) {
2878 Add->setOperand(i_nocapture: 1, Val_nocapture: CastedTaskLB);
2879 }
2880 }
2881 }
2882 }
2883 }
2884 }
2885 }
2886
2887 FakeLB->replaceAllUsesWith(V: CastedLBVal);
2888 FakeUB->replaceAllUsesWith(V: CastedUBVal);
2889 FakeStep->replaceAllUsesWith(V: CastedStepVal);
2890 for (Instruction *I : llvm::reverse(C&: ToBeDeleted)) {
2891 I->eraseFromParent();
2892 }
2893 };
2894
2895 addOutlineInfo(OI: std::move(OI));
2896 Builder.SetInsertPoint(TaskloopExitBB->begin());
2897 return Builder.saveIP();
2898}
2899
2900llvm::StructType *OpenMPIRBuilder::getKmpTaskAffinityInfoTy() {
2901 llvm::Type *IntPtrTy = llvm::Type::getIntNTy(
2902 C&: M.getContext(), N: M.getDataLayout().getPointerSizeInBits());
2903 return llvm::StructType::get(elt1: IntPtrTy, elts: IntPtrTy,
2904 elts: llvm::Type::getInt32Ty(C&: M.getContext()));
2905}
2906
2907OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTask(
2908 const LocationDescription &Loc, InsertPointTy AllocaIP,
2909 ArrayRef<BasicBlock *> DeallocBlocks, BodyGenCallbackTy BodyGenCB,
2910 bool Tied, Value *Final, Value *IfCondition,
2911 const DependenciesInfo &Dependencies, const AffinityData &Affinities,
2912 bool Mergeable, Value *EventHandle, Value *Priority, bool FreeAgent) {
2913
2914 if (!updateToLocation(Loc))
2915 return InsertPointTy();
2916
2917 uint32_t SrcLocStrSize;
2918 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2919 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2920 // The current basic block is split into four basic blocks. After outlining,
2921 // they will be mapped as follows:
2922 // ```
2923 // def current_fn() {
2924 // current_basic_block:
2925 // br label %task.exit
2926 // task.exit:
2927 // ; instructions after task
2928 // }
2929 // def outlined_fn() {
2930 // task.alloca:
2931 // br label %task.body
2932 // task.body:
2933 // ret void
2934 // }
2935 // ```
2936 BasicBlock *TaskExitBB = splitBB(Builder, /*CreateBranch=*/true, Name: "task.exit");
2937 BasicBlock *TaskBodyBB = splitBB(Builder, /*CreateBranch=*/true, Name: "task.body");
2938 BasicBlock *TaskAllocaBB =
2939 splitBB(Builder, /*CreateBranch=*/true, Name: "task.alloca");
2940
2941 InsertPointTy TaskAllocaIP = TaskAllocaBB->begin();
2942 InsertPointTy TaskBodyIP = TaskBodyBB->begin();
2943 if (Error Err = BodyGenCB(TaskAllocaIP, TaskBodyIP, TaskExitBB))
2944 return Err;
2945
2946 auto OI = std::make_unique<OutlineInfo>();
2947 OI->EntryBB = TaskAllocaBB;
2948 OI->OuterAllocBB = AllocaIP.getNodeParent();
2949 OI->ExitBB = TaskExitBB;
2950 OI->OuterDeallocBBs.reserve(N: DeallocBlocks.size());
2951 copy(Range&: DeallocBlocks, Out: OI->OuterDeallocBBs.end());
2952
2953 // Add the thread ID argument.
2954 SmallVector<Instruction *, 4> ToBeDeleted;
2955 OI->ExcludeArgsFromAggregate.push_back(Elt: createFakeIntVal(
2956 Builder, OuterAllocaIP: AllocaIP, ToBeDeleted, InnerAllocaIP: TaskAllocaIP, Name: "global.tid", AsPtr: false));
2957
2958 OI->PostOutlineCB = [this, Ident, Tied, Final, IfCondition, Dependencies,
2959 Affinities, Mergeable, Priority, EventHandle, FreeAgent,
2960 TaskAllocaBB,
2961 ToBeDeleted](Function &OutlinedFn) mutable {
2962 // Replace the Stale CI by appropriate RTL function call.
2963 assert(OutlinedFn.hasOneUse() &&
2964 "there must be a single user for the outlined function");
2965 CallInst *StaleCI = cast<CallInst>(Val: OutlinedFn.user_back());
2966
2967 // HasShareds is true if any variables are captured in the outlined region,
2968 // false otherwise.
2969 bool HasShareds = StaleCI->arg_size() > 1;
2970 Builder.SetInsertPoint(StaleCI);
2971
2972 // Gather the arguments for emitting the runtime call for
2973 // @__kmpc_omp_task_alloc
2974 Function *TaskAllocFn =
2975 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_alloc);
2976
2977 // Arguments - `loc_ref` (Ident) and `gtid` (ThreadID)
2978 // call.
2979 Value *ThreadID = getOrCreateThreadID(Ident);
2980
2981 // Argument - `flags`
2982 // Task is tied iff (Flags & 1) == 1.
2983 // Task is untied iff (Flags & 1) == 0.
2984 // Task is final iff (Flags & 2) == 2.
2985 // Task is not final iff (Flags & 2) == 0.
2986 // Task is mergeable or merged-if0 iff (Flags & 4) == 4.
2987 // Task is neither mergeable nor merged-if0 iff (Flags & 4) == 0.
2988 // Task is detachable iff (Flags & 64) == 64.
2989 // Task is not detachable iff (Flags & 64) == 0.
2990 // Task is priority iff (Flags & 32) == 32.
2991 // Task is not priority iff (Flags & 32) == 0.
2992 // Task is free-agent eligible iff (Flags & 128) == 128.
2993 // Task is not free-agent eligible iff (Flags & 128) == 0.
2994 // TODO: Handle the other flags.
2995 Value *Flags = Builder.getInt32(C: Tied);
2996 auto *ConstIfCondition = dyn_cast_or_null<ConstantInt>(Val: IfCondition);
2997 bool UseMergedIf0Path = ConstIfCondition && ConstIfCondition->isZero();
2998 if (Final) {
2999 Value *FinalFlag =
3000 Builder.CreateSelect(C: Final, True: Builder.getInt32(C: 2), False: Builder.getInt32(C: 0));
3001 Flags = Builder.CreateOr(LHS: FinalFlag, RHS: Flags);
3002 }
3003
3004 if (Mergeable || UseMergedIf0Path)
3005 Flags = Builder.CreateOr(LHS: Builder.getInt32(C: 4), RHS: Flags);
3006 if (EventHandle)
3007 Flags = Builder.CreateOr(LHS: Builder.getInt32(C: 64), RHS: Flags);
3008 if (Priority)
3009 Flags = Builder.CreateOr(LHS: Builder.getInt32(C: 32), RHS: Flags);
3010 if (FreeAgent)
3011 Flags = Builder.CreateOr(LHS: Builder.getInt32(C: 128), RHS: Flags);
3012
3013 // Argument - `sizeof_kmp_task_t` (TaskSize)
3014 // Tasksize refers to the size in bytes of kmp_task_t data structure
3015 // including private vars accessed in task.
3016 // TODO: add kmp_task_t_with_privates (privates)
3017 Value *TaskSize = Builder.getInt64(
3018 C: divideCeil(Numerator: M.getDataLayout().getTypeSizeInBits(Ty: Task), Denominator: 8));
3019
3020 // Argument - `sizeof_shareds` (SharedsSize)
3021 // SharedsSize refers to the shareds array size in the kmp_task_t data
3022 // structure.
3023 Value *SharedsSize = Builder.getInt64(C: 0);
3024 if (HasShareds) {
3025 AllocaInst *ArgStructAlloca =
3026 dyn_cast<AllocaInst>(Val: StaleCI->getArgOperand(i: 1));
3027 assert(ArgStructAlloca &&
3028 "Unable to find the alloca instruction corresponding to arguments "
3029 "for extracted function");
3030 std::optional<TypeSize> ArgAllocSize =
3031 ArgStructAlloca->getAllocationSize(DL: M.getDataLayout());
3032 assert(ArgAllocSize &&
3033 "Unable to determine size of arguments for extracted function");
3034 SharedsSize = Builder.getInt64(C: ArgAllocSize->getFixedValue());
3035 }
3036 // Emit the @__kmpc_omp_task_alloc runtime call
3037 // The runtime call returns a pointer to an area where the task captured
3038 // variables must be copied before the task is run (TaskData)
3039 CallInst *TaskData = createRuntimeFunctionCall(
3040 Callee: TaskAllocFn, Args: {/*loc_ref=*/Ident, /*gtid=*/ThreadID, /*flags=*/Flags,
3041 /*sizeof_task=*/TaskSize, /*sizeof_shared=*/SharedsSize,
3042 /*task_func=*/&OutlinedFn});
3043
3044 if (Affinities.Count && Affinities.Info) {
3045 Function *RegAffFn = getOrCreateRuntimeFunctionPtr(
3046 FnID: OMPRTL___kmpc_omp_reg_task_with_affinity);
3047
3048 createRuntimeFunctionCall(Callee: RegAffFn, Args: {Ident, ThreadID, TaskData,
3049 Affinities.Count, Affinities.Info});
3050 }
3051
3052 // Emit detach clause initialization.
3053 // evt = (typeof(evt))__kmpc_task_allow_completion_event(loc, tid,
3054 // task_descriptor);
3055 if (EventHandle) {
3056 Function *TaskDetachFn = getOrCreateRuntimeFunctionPtr(
3057 FnID: OMPRTL___kmpc_task_allow_completion_event);
3058 llvm::Value *EventVal =
3059 createRuntimeFunctionCall(Callee: TaskDetachFn, Args: {Ident, ThreadID, TaskData});
3060 llvm::Value *EventHandleAddr =
3061 Builder.CreatePointerBitCastOrAddrSpaceCast(V: EventHandle,
3062 DestTy: Builder.getPtrTy(AddrSpace: 0));
3063 EventVal = Builder.CreatePtrToInt(V: EventVal, DestTy: Builder.getInt64Ty());
3064 Builder.CreateStore(Val: EventVal, Ptr: EventHandleAddr);
3065 }
3066 // Copy the arguments for outlined function
3067 if (HasShareds) {
3068 Value *Shareds = StaleCI->getArgOperand(i: 1);
3069 Align Alignment = TaskData->getPointerAlignment(DL: M.getDataLayout());
3070 Value *TaskShareds = Builder.CreateLoad(Ty: VoidPtr, Ptr: TaskData);
3071 Builder.CreateMemCpy(Dst: TaskShareds, DstAlign: Alignment, Src: Shareds, SrcAlign: Alignment,
3072 Size: SharedsSize);
3073 }
3074
3075 if (Priority) {
3076 //
3077 // The return type of "__kmpc_omp_task_alloc" is "kmp_task_t *",
3078 // we populate the priority information into the "kmp_task_t" here
3079 //
3080 // The struct "kmp_task_t" definition is available in kmp.h
3081 // kmp_task_t = { shareds, routine, part_id, data1, data2 }
3082 // data2 is used for priority
3083 //
3084 Type *Int32Ty = Builder.getInt32Ty();
3085 Constant *Zero = ConstantInt::get(Ty: Int32Ty, V: 0);
3086 // kmp_task_t* => { ptr }
3087 Type *TaskPtr = StructType::get(elt1: VoidPtr);
3088 Value *TaskGEP =
3089 Builder.CreateInBoundsGEP(Ty: TaskPtr, Ptr: TaskData, IdxList: {Zero, Zero});
3090 // kmp_task_t => { ptr, ptr, i32, ptr, ptr }
3091 Type *TaskStructType = StructType::get(
3092 elt1: VoidPtr, elts: VoidPtr, elts: Builder.getInt32Ty(), elts: VoidPtr, elts: VoidPtr);
3093 Value *PriorityData = Builder.CreateInBoundsGEP(
3094 Ty: TaskStructType, Ptr: TaskGEP, IdxList: {Zero, ConstantInt::get(Ty: Int32Ty, V: 4)});
3095 // kmp_cmplrdata_t => { ptr, ptr }
3096 Type *CmplrStructType = StructType::get(elt1: VoidPtr, elts: VoidPtr);
3097 Value *CmplrData = Builder.CreateInBoundsGEP(Ty: CmplrStructType,
3098 Ptr: PriorityData, IdxList: {Zero, Zero});
3099 Builder.CreateStore(Val: Priority, Ptr: CmplrData);
3100 }
3101
3102 Value *DepArray = nullptr;
3103 Value *NumDeps = nullptr;
3104 if (Dependencies.DepArray) {
3105 DepArray = Dependencies.DepArray;
3106 NumDeps = Dependencies.NumDeps;
3107 } else if (!Dependencies.Deps.empty()) {
3108 DepArray = emitTaskDependencies(OMPBuilder&: *this, Dependencies: Dependencies.Deps);
3109 NumDeps = Builder.getInt32(C: Dependencies.Deps.size());
3110 }
3111
3112 // In the presence of the `if` clause, the following IR is generated:
3113 // ...
3114 // %data = call @__kmpc_omp_task_alloc(...)
3115 // br i1 %if_condition, label %then, label %else
3116 // then:
3117 // call @__kmpc_omp_task(...)
3118 // br label %exit
3119 // else:
3120 // ;; Wait for resolution of dependencies, if any, before
3121 // ;; beginning the task
3122 // call @__kmpc_omp_wait_deps(...)
3123 // call @__kmpc_omp_task_begin_if0(...)
3124 // call @outlined_fn(...)
3125 // call @__kmpc_omp_task_complete_if0(...)
3126 // br label %exit
3127 // exit:
3128 // ...
3129 if (IfCondition && !UseMergedIf0Path) {
3130 // `SplitBlockAndInsertIfThenElse` requires the block to have a
3131 // terminator.
3132 splitBB(Builder, /*CreateBranch=*/true, Name: "if.end");
3133 Instruction *IfTerminator =
3134 Builder.GetInsertPoint()->getParent()->getTerminator();
3135 Instruction *ThenTI = IfTerminator, *ElseTI = nullptr;
3136 Builder.SetInsertPoint(IfTerminator);
3137 SplitBlockAndInsertIfThenElse(Cond: IfCondition, SplitBefore: IfTerminator, ThenTerm: &ThenTI,
3138 ElseTerm: &ElseTI);
3139 Builder.SetInsertPoint(ElseTI);
3140
3141 if (DepArray) {
3142 Function *TaskWaitFn =
3143 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_wait_deps);
3144 createRuntimeFunctionCall(
3145 Callee: TaskWaitFn,
3146 Args: {Ident, ThreadID, NumDeps, DepArray,
3147 ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0),
3148 ConstantPointerNull::get(T: PointerType::getUnqual(C&: M.getContext()))});
3149 }
3150 Function *TaskBeginFn =
3151 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_begin_if0);
3152 Function *TaskCompleteFn =
3153 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_complete_if0);
3154 createRuntimeFunctionCall(Callee: TaskBeginFn, Args: {Ident, ThreadID, TaskData});
3155 CallInst *CI = nullptr;
3156 if (HasShareds)
3157 CI = createRuntimeFunctionCall(Callee: &OutlinedFn, Args: {ThreadID, TaskData});
3158 else
3159 CI = createRuntimeFunctionCall(Callee: &OutlinedFn, Args: {ThreadID});
3160 CI->setDebugLoc(StaleCI->getDebugLoc());
3161 createRuntimeFunctionCall(Callee: TaskCompleteFn, Args: {Ident, ThreadID, TaskData});
3162 Builder.SetInsertPoint(ThenTI);
3163 }
3164
3165 if (DepArray) {
3166 Function *TaskFn =
3167 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_with_deps);
3168 createRuntimeFunctionCall(
3169 Callee: TaskFn,
3170 Args: {Ident, ThreadID, TaskData, NumDeps, DepArray,
3171 ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0),
3172 ConstantPointerNull::get(T: PointerType::getUnqual(C&: M.getContext()))});
3173
3174 } else {
3175 // Emit the @__kmpc_omp_task runtime call to spawn the task
3176 Function *TaskFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task);
3177 createRuntimeFunctionCall(Callee: TaskFn, Args: {Ident, ThreadID, TaskData});
3178 }
3179
3180 StaleCI->eraseFromParent();
3181
3182 Builder.SetInsertPoint(TaskAllocaBB->begin());
3183 if (HasShareds) {
3184 LoadInst *Shareds = Builder.CreateLoad(Ty: VoidPtr, Ptr: OutlinedFn.getArg(i: 1));
3185 OutlinedFn.getArg(i: 1)->replaceUsesWithIf(
3186 New: Shareds, ShouldReplace: [Shareds](Use &U) { return U.getUser() != Shareds; });
3187 }
3188
3189 // The insert point may refer to one of the instructions about to be
3190 // deleted. It is not needed anymore so clear it instead of leaving it
3191 // dangling.
3192 Builder.ClearInsertionPoint();
3193 for (Instruction *I : llvm::reverse(C&: ToBeDeleted))
3194 I->eraseFromParent();
3195 };
3196
3197 addOutlineInfo(OI: std::move(OI));
3198 Builder.SetInsertPoint(TaskExitBB->begin());
3199
3200 return Builder.saveIP();
3201}
3202
3203OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTaskgroup(
3204 const LocationDescription &Loc, InsertPointTy AllocaIP,
3205 ArrayRef<BasicBlock *> DeallocBlocks, BodyGenCallbackTy BodyGenCB) {
3206 if (!updateToLocation(Loc))
3207 return InsertPointTy();
3208
3209 uint32_t SrcLocStrSize;
3210 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
3211 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
3212 Value *ThreadID = getOrCreateThreadID(Ident);
3213
3214 // Emit the @__kmpc_taskgroup runtime call to start the taskgroup
3215 Function *TaskgroupFn =
3216 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_taskgroup);
3217 createRuntimeFunctionCall(Callee: TaskgroupFn, Args: {Ident, ThreadID});
3218
3219 BasicBlock *TaskgroupExitBB = splitBB(Builder, CreateBranch: true, Name: "taskgroup.exit");
3220 if (Error Err = BodyGenCB(AllocaIP, Builder.saveIP(), DeallocBlocks))
3221 return Err;
3222
3223 Builder.SetInsertPoint(TaskgroupExitBB);
3224 // Emit the @__kmpc_end_taskgroup runtime call to end the taskgroup
3225 Function *EndTaskgroupFn =
3226 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_end_taskgroup);
3227 createRuntimeFunctionCall(Callee: EndTaskgroupFn, Args: {Ident, ThreadID});
3228
3229 return Builder.saveIP();
3230}
3231
3232OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createSections(
3233 const LocationDescription &Loc, InsertPointTy AllocaIP,
3234 ArrayRef<StorableBodyGenCallbackTy> SectionCBs, PrivatizeCallbackTy PrivCB,
3235 FinalizeCallbackTy FiniCB, bool IsCancellable, bool IsNowait) {
3236 assert(!isConflictIP(AllocaIP, Loc.IP) && "Dedicated IP allocas required");
3237
3238 if (!updateToLocation(Loc))
3239 return Loc.IP;
3240
3241 FinalizationStack.push_back(Elt: {FiniCB, OMPD_sections, IsCancellable});
3242
3243 // Each section is emitted as a switch case
3244 // Each finalization callback is handled from clang.EmitOMPSectionDirective()
3245 // -> OMP.createSection() which generates the IR for each section
3246 // Iterate through all sections and emit a switch construct:
3247 // switch (IV) {
3248 // case 0:
3249 // <SectionStmt[0]>;
3250 // break;
3251 // ...
3252 // case <NumSection> - 1:
3253 // <SectionStmt[<NumSection> - 1]>;
3254 // break;
3255 // }
3256 // ...
3257 // section_loop.after:
3258 // <FiniCB>;
3259 auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, Value *IndVar) -> Error {
3260 Builder.restoreIP(IP: CodeGenIP);
3261 BasicBlock *Continue =
3262 splitBBWithSuffix(Builder, /*CreateBranch=*/false, Suffix: ".sections.after");
3263 Function *CurFn = Continue->getParent();
3264 SwitchInst *SwitchStmt = Builder.CreateSwitch(V: IndVar, Dest: Continue);
3265
3266 unsigned CaseNumber = 0;
3267 for (auto SectionCB : SectionCBs) {
3268 BasicBlock *CaseBB = BasicBlock::Create(
3269 Context&: M.getContext(), Name: "omp_section_loop.body.case", Parent: CurFn, InsertBefore: Continue);
3270 SwitchStmt->addCase(OnVal: Builder.getInt32(C: CaseNumber), Dest: CaseBB);
3271 Builder.SetInsertPoint(CaseBB);
3272 UncondBrInst *CaseEndBr = Builder.CreateBr(Dest: Continue);
3273 if (Error Err = SectionCB(InsertPointTy(), CaseEndBr->getIterator(), {}))
3274 return Err;
3275 CaseNumber++;
3276 }
3277 // remove the existing terminator from body BB since there can be no
3278 // terminators after switch/case
3279 return Error::success();
3280 };
3281 // Loop body ends here
3282 // LowerBound, UpperBound, and STride for createCanonicalLoop
3283 Type *I32Ty = Type::getInt32Ty(C&: M.getContext());
3284 Value *LB = ConstantInt::get(Ty: I32Ty, V: 0);
3285 Value *UB = ConstantInt::get(Ty: I32Ty, V: SectionCBs.size());
3286 Value *ST = ConstantInt::get(Ty: I32Ty, V: 1);
3287 Expected<CanonicalLoopInfo *> LoopInfo = createCanonicalLoop(
3288 Loc, BodyGenCB: LoopBodyGenCB, Start: LB, Stop: UB, Step: ST, IsSigned: true, InclusiveStop: false, ComputeIP: AllocaIP, Name: "section_loop");
3289 if (!LoopInfo)
3290 return LoopInfo.takeError();
3291
3292 InsertPointOrErrorTy WsloopIP =
3293 applyStaticWorkshareLoop(DL: Loc.DL, CLI: *LoopInfo, AllocaIP,
3294 LoopType: WorksharingLoopType::ForStaticLoop, NeedsBarrier: !IsNowait);
3295 if (!WsloopIP)
3296 return WsloopIP.takeError();
3297 InsertPointTy AfterIP = *WsloopIP;
3298
3299 BasicBlock *LoopFini = AfterIP.getNodeParent()->getSinglePredecessor();
3300 assert(LoopFini && "Bad structure of static workshare loop finalization");
3301
3302 // Apply the finalization callback in LoopAfterBB
3303 auto FiniInfo = FinalizationStack.pop_back_val();
3304 assert(FiniInfo.DK == OMPD_sections &&
3305 "Unexpected finalization stack state!");
3306 if (Error Err = FiniInfo.mergeFiniBB(Builder, OtherFiniBB: LoopFini))
3307 return Err;
3308
3309 return AfterIP;
3310}
3311
3312OpenMPIRBuilder::InsertPointOrErrorTy
3313OpenMPIRBuilder::createSection(const LocationDescription &Loc,
3314 BodyGenCallbackTy BodyGenCB,
3315 FinalizeCallbackTy FiniCB) {
3316 if (!updateToLocation(Loc))
3317 return Loc.IP;
3318
3319 auto FiniCBWrapper = [&](InsertPointTy IP) {
3320 if (IP != IP.getNodeParent()->end())
3321 return FiniCB(IP);
3322 // This must be done otherwise any nested constructs using FinalizeOMPRegion
3323 // will fail because that function requires the Finalization Basic Block to
3324 // have a terminator, which is already removed by EmitOMPRegionBody.
3325 // IP is currently at cancelation block.
3326 // We need to backtrack to the condition block to fetch
3327 // the exit block and create a branch from cancelation
3328 // to exit block.
3329 IRBuilder<>::InsertPointGuard IPG(Builder);
3330 Builder.restoreIP(IP);
3331 auto *CaseBB = Loc.IP.getNodeParent();
3332 auto *CondBB = CaseBB->getSinglePredecessor()->getSinglePredecessor();
3333 auto *ExitBB = CondBB->getTerminator()->getSuccessor(Idx: 1);
3334 Instruction *I = Builder.CreateBr(Dest: ExitBB);
3335 IP = I->getIterator();
3336 return FiniCB(IP);
3337 };
3338
3339 Directive OMPD = Directive::OMPD_sections;
3340 // Since we are using Finalization Callback here, HasFinalize
3341 // and IsCancellable have to be true
3342 return EmitOMPInlinedRegion(OMPD, EntryCall: nullptr, ExitCall: nullptr, BodyGenCB, FiniCB: FiniCBWrapper,
3343 /*Conditional*/ false, /*hasFinalize*/ HasFinalize: true,
3344 /*IsCancellable*/ true);
3345}
3346
3347static OpenMPIRBuilder::InsertPointTy getInsertPointAfterInstr(Instruction *I) {
3348 BasicBlock::iterator IT(I);
3349 IT++;
3350 return IT;
3351}
3352
3353Value *OpenMPIRBuilder::getGPUThreadID() {
3354 return createRuntimeFunctionCall(
3355 Callee: getOrCreateRuntimeFunction(M,
3356 FnID: OMPRTL___kmpc_get_hardware_thread_id_in_block),
3357 Args: {});
3358}
3359
3360Value *OpenMPIRBuilder::getGPUWarpSize() {
3361 return createRuntimeFunctionCall(
3362 Callee: getOrCreateRuntimeFunction(M, FnID: OMPRTL___kmpc_get_warp_size), Args: {});
3363}
3364
3365Value *OpenMPIRBuilder::getNVPTXWarpID() {
3366 unsigned LaneIDBits = Log2_32(Value: Config.getGridValue().GV_Warp_Size);
3367 return Builder.CreateAShr(LHS: getGPUThreadID(), RHS: LaneIDBits, Name: "nvptx_warp_id");
3368}
3369
3370Value *OpenMPIRBuilder::getNVPTXLaneID() {
3371 unsigned LaneIDBits = Log2_32(Value: Config.getGridValue().GV_Warp_Size);
3372 assert(LaneIDBits < 32 && "Invalid LaneIDBits size in NVPTX device.");
3373 unsigned LaneIDMask = ~0u >> (32u - LaneIDBits);
3374 return Builder.CreateAnd(LHS: getGPUThreadID(), RHS: Builder.getInt32(C: LaneIDMask),
3375 Name: "nvptx_lane_id");
3376}
3377
3378Value *OpenMPIRBuilder::castValueToType(InsertPointTy AllocaIP, Value *From,
3379 Type *ToType) {
3380 Type *FromType = From->getType();
3381 uint64_t FromSize = M.getDataLayout().getTypeStoreSize(Ty: FromType);
3382 uint64_t ToSize = M.getDataLayout().getTypeStoreSize(Ty: ToType);
3383 assert(FromSize > 0 && "From size must be greater than zero");
3384 assert(ToSize > 0 && "To size must be greater than zero");
3385 if (FromType == ToType)
3386 return From;
3387 if (FromSize == ToSize)
3388 return Builder.CreateBitCast(V: From, DestTy: ToType);
3389 if (ToType->isIntegerTy() && FromType->isIntegerTy())
3390 return Builder.CreateIntCast(V: From, DestTy: ToType, /*isSigned*/ true);
3391 InsertPointTy SaveIP = Builder.saveIP();
3392 Builder.restoreIP(IP: AllocaIP);
3393 Value *CastItem = Builder.CreateAlloca(Ty: ToType);
3394 Builder.restoreIP(IP: SaveIP);
3395
3396 Value *ValCastItem = Builder.CreatePointerBitCastOrAddrSpaceCast(
3397 V: CastItem, DestTy: Builder.getPtrTy(AddrSpace: 0));
3398 Builder.CreateStore(Val: From, Ptr: ValCastItem);
3399 return Builder.CreateLoad(Ty: ToType, Ptr: CastItem);
3400}
3401
3402Value *OpenMPIRBuilder::createRuntimeShuffleFunction(InsertPointTy AllocaIP,
3403 Value *Element,
3404 Type *ElementType,
3405 Value *Offset) {
3406 uint64_t Size = M.getDataLayout().getTypeStoreSize(Ty: ElementType);
3407 assert(Size <= 8 && "Unsupported bitwidth in shuffle instruction");
3408
3409 // Cast all types to 32- or 64-bit values before calling shuffle routines.
3410 Type *CastTy = Builder.getIntNTy(N: Size <= 4 ? 32 : 64);
3411 Value *ElemCast = castValueToType(AllocaIP, From: Element, ToType: CastTy);
3412 Value *WarpSize =
3413 Builder.CreateIntCast(V: getGPUWarpSize(), DestTy: Builder.getInt16Ty(), isSigned: true);
3414 Function *ShuffleFunc = getOrCreateRuntimeFunctionPtr(
3415 FnID: Size <= 4 ? RuntimeFunction::OMPRTL___kmpc_shuffle_int32
3416 : RuntimeFunction::OMPRTL___kmpc_shuffle_int64);
3417 Value *WarpSizeCast =
3418 Builder.CreateIntCast(V: WarpSize, DestTy: Builder.getInt16Ty(), /*isSigned=*/true);
3419 Value *ShuffleCall =
3420 createRuntimeFunctionCall(Callee: ShuffleFunc, Args: {ElemCast, Offset, WarpSizeCast});
3421 // The shuffle runtime functions return a 32- or 64-bit value. Cast it back
3422 // down to the requested element type, otherwise storing the result would
3423 // write past the end of an element narrower than the shuffle width.
3424 return castValueToType(AllocaIP, From: ShuffleCall, ToType: ElementType);
3425}
3426
3427void OpenMPIRBuilder::shuffleAndStore(InsertPointTy AllocaIP, Value *SrcAddr,
3428 Value *DstAddr, Type *ElemType,
3429 Value *Offset, Type *ReductionArrayTy,
3430 bool IsByRefElem) {
3431 uint64_t Size = M.getDataLayout().getTypeStoreSize(Ty: ElemType);
3432 // Create the loop over the big sized data.
3433 // ptr = (void*)Elem;
3434 // ptrEnd = (void*) Elem + 1;
3435 // Step = 8;
3436 // while (ptr + Step < ptrEnd)
3437 // shuffle((int64_t)*ptr);
3438 // Step = 4;
3439 // while (ptr + Step < ptrEnd)
3440 // shuffle((int32_t)*ptr);
3441 // ...
3442 Type *IndexTy = Builder.getIndexTy(
3443 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
3444 Value *ElemPtr = DstAddr;
3445 Value *Ptr = SrcAddr;
3446 for (unsigned IntSize = 8; IntSize >= 1; IntSize /= 2) {
3447 if (Size < IntSize)
3448 continue;
3449 Type *IntType = Builder.getIntNTy(N: IntSize * 8);
3450 Ptr = Builder.CreatePointerBitCastOrAddrSpaceCast(
3451 V: Ptr, DestTy: Builder.getPtrTy(AddrSpace: 0), Name: Ptr->getName() + ".ascast");
3452 Value *SrcAddrGEP =
3453 Builder.CreateGEP(Ty: ElemType, Ptr: SrcAddr, IdxList: {ConstantInt::get(Ty: IndexTy, V: 1)});
3454 ElemPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
3455 V: ElemPtr, DestTy: Builder.getPtrTy(AddrSpace: 0), Name: ElemPtr->getName() + ".ascast");
3456
3457 Function *CurFunc = Builder.GetInsertBlock()->getParent();
3458 if ((Size / IntSize) > 1) {
3459 Value *PtrEnd = Builder.CreatePointerBitCastOrAddrSpaceCast(
3460 V: SrcAddrGEP, DestTy: Builder.getPtrTy());
3461 BasicBlock *PreCondBB =
3462 BasicBlock::Create(Context&: M.getContext(), Name: ".shuffle.pre_cond");
3463 BasicBlock *ThenBB = BasicBlock::Create(Context&: M.getContext(), Name: ".shuffle.then");
3464 BasicBlock *ExitBB = BasicBlock::Create(Context&: M.getContext(), Name: ".shuffle.exit");
3465 BasicBlock *CurrentBB = Builder.GetInsertBlock();
3466 emitBlock(BB: PreCondBB, CurFn: CurFunc);
3467 PHINode *PhiSrc =
3468 Builder.CreatePHI(Ty: Ptr->getType(), /*NumReservedValues=*/2);
3469 PhiSrc->addIncoming(V: Ptr, BB: CurrentBB);
3470 PHINode *PhiDest =
3471 Builder.CreatePHI(Ty: ElemPtr->getType(), /*NumReservedValues=*/2);
3472 PhiDest->addIncoming(V: ElemPtr, BB: CurrentBB);
3473 Ptr = PhiSrc;
3474 ElemPtr = PhiDest;
3475 Value *PtrDiff = Builder.CreatePtrDiff(
3476 ElemTy: Builder.getInt8Ty(), LHS: PtrEnd,
3477 RHS: Builder.CreatePointerBitCastOrAddrSpaceCast(V: Ptr, DestTy: Builder.getPtrTy()));
3478 Builder.CreateCondBr(
3479 Cond: Builder.CreateICmpSGT(LHS: PtrDiff, RHS: Builder.getInt64(C: IntSize - 1)), True: ThenBB,
3480 False: ExitBB);
3481 emitBlock(BB: ThenBB, CurFn: CurFunc);
3482 Value *Res = createRuntimeShuffleFunction(
3483 AllocaIP,
3484 Element: Builder.CreateAlignedLoad(
3485 Ty: IntType, Ptr, Align: M.getDataLayout().getPrefTypeAlign(Ty: ElemType)),
3486 ElementType: IntType, Offset);
3487 Builder.CreateAlignedStore(Val: Res, Ptr: ElemPtr,
3488 Align: M.getDataLayout().getPrefTypeAlign(Ty: ElemType));
3489 Value *LocalPtr =
3490 Builder.CreateGEP(Ty: IntType, Ptr, IdxList: {ConstantInt::get(Ty: IndexTy, V: 1)});
3491 Value *LocalElemPtr =
3492 Builder.CreateGEP(Ty: IntType, Ptr: ElemPtr, IdxList: {ConstantInt::get(Ty: IndexTy, V: 1)});
3493 PhiSrc->addIncoming(V: LocalPtr, BB: ThenBB);
3494 PhiDest->addIncoming(V: LocalElemPtr, BB: ThenBB);
3495 emitBranch(Target: PreCondBB);
3496 emitBlock(BB: ExitBB, CurFn: CurFunc);
3497 } else {
3498 // The shuffled value comes back as the chunk's integer type, so the
3499 // store covers exactly this chunk regardless of what ElemType is.
3500 Value *Res = createRuntimeShuffleFunction(
3501 AllocaIP, Element: Builder.CreateLoad(Ty: IntType, Ptr), ElementType: IntType, Offset);
3502 Builder.CreateStore(Val: Res, Ptr: ElemPtr);
3503 Ptr = Builder.CreateGEP(Ty: IntType, Ptr, IdxList: {ConstantInt::get(Ty: IndexTy, V: 1)});
3504 ElemPtr =
3505 Builder.CreateGEP(Ty: IntType, Ptr: ElemPtr, IdxList: {ConstantInt::get(Ty: IndexTy, V: 1)});
3506 }
3507 Size = Size % IntSize;
3508 }
3509}
3510
3511Error OpenMPIRBuilder::emitReductionListCopy(
3512 InsertPointTy AllocaIP, CopyAction Action, Type *ReductionArrayTy,
3513 ArrayRef<ReductionInfo> ReductionInfos, Value *SrcBase, Value *DestBase,
3514 ArrayRef<bool> IsByRef, CopyOptionsTy CopyOptions) {
3515 Type *IndexTy = Builder.getIndexTy(
3516 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
3517 Value *RemoteLaneOffset = CopyOptions.RemoteLaneOffset;
3518
3519 // Iterates, element-by-element, through the source Reduce list and
3520 // make a copy.
3521 for (auto En : enumerate(First&: ReductionInfos)) {
3522 const ReductionInfo &RI = En.value();
3523 Value *SrcElementAddr = nullptr;
3524 AllocaInst *DestAlloca = nullptr;
3525 Value *DestElementAddr = nullptr;
3526 Value *DestElementPtrAddr = nullptr;
3527 // Should we shuffle in an element from a remote lane?
3528 bool ShuffleInElement = false;
3529 // Set to true to update the pointer in the dest Reduce list to a
3530 // newly created element.
3531 bool UpdateDestListPtr = false;
3532
3533 // Step 1.1: Get the address for the src element in the Reduce list.
3534 Value *SrcElementPtrAddr = Builder.CreateInBoundsGEP(
3535 Ty: ReductionArrayTy, Ptr: SrcBase,
3536 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
3537 SrcElementAddr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: SrcElementPtrAddr);
3538
3539 // Step 1.2: Create a temporary to store the element in the destination
3540 // Reduce list.
3541 DestElementPtrAddr = Builder.CreateInBoundsGEP(
3542 Ty: ReductionArrayTy, Ptr: DestBase,
3543 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
3544 bool IsByRefElem = (!IsByRef.empty() && IsByRef[En.index()]);
3545 switch (Action) {
3546 case CopyAction::RemoteLaneToThread: {
3547 InsertPointTy CurIP = Builder.saveIP();
3548 Builder.restoreIP(IP: AllocaIP);
3549
3550 Type *DestAllocaType =
3551 IsByRefElem ? RI.ByRefAllocatedType : RI.ElementType;
3552 DestAlloca = Builder.CreateAlloca(Ty: DestAllocaType, ArraySize: nullptr,
3553 Name: ".omp.reduction.element");
3554 DestAlloca->setAlignment(
3555 M.getDataLayout().getPrefTypeAlign(Ty: DestAllocaType));
3556 DestElementAddr = DestAlloca;
3557 DestElementAddr =
3558 Builder.CreateAddrSpaceCast(V: DestElementAddr, DestTy: Builder.getPtrTy(),
3559 Name: DestElementAddr->getName() + ".ascast");
3560 Builder.restoreIP(IP: CurIP);
3561 ShuffleInElement = true;
3562 UpdateDestListPtr = true;
3563 break;
3564 }
3565 case CopyAction::ThreadCopy: {
3566 DestElementAddr =
3567 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: DestElementPtrAddr);
3568 break;
3569 }
3570 }
3571
3572 // Now that all active lanes have read the element in the
3573 // Reduce list, shuffle over the value from the remote lane.
3574 if (ShuffleInElement) {
3575 Type *ShuffleType = RI.ElementType;
3576 Value *ShuffleSrcAddr = SrcElementAddr;
3577 Value *ShuffleDestAddr = DestElementAddr;
3578 AllocaInst *LocalStorage = nullptr;
3579
3580 if (IsByRefElem) {
3581 assert(RI.ByRefElementType && "Expected by-ref element type to be set");
3582 assert(RI.ByRefAllocatedType &&
3583 "Expected by-ref allocated type to be set");
3584 // For by-ref reductions, we need to copy from the remote lane the
3585 // actual value of the partial reduction computed by that remote lane;
3586 // rather than, for example, a pointer to that data or, even worse, a
3587 // pointer to the descriptor of the by-ref reduction element.
3588 ShuffleType = RI.ByRefElementType;
3589
3590 if (RI.DataPtrPtrGen) {
3591 // Descriptor-based by-ref: extract data pointer from descriptor.
3592 InsertPointOrErrorTy GenResult = RI.DataPtrPtrGen(
3593 Builder.saveIP(), ShuffleSrcAddr, ShuffleSrcAddr);
3594
3595 if (!GenResult)
3596 return GenResult.takeError();
3597
3598 ShuffleSrcAddr =
3599 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ShuffleSrcAddr);
3600
3601 {
3602 InsertPointTy OldIP = Builder.saveIP();
3603 Builder.restoreIP(IP: AllocaIP);
3604
3605 LocalStorage = Builder.CreateAlloca(Ty: ShuffleType);
3606 Builder.restoreIP(IP: OldIP);
3607 ShuffleDestAddr = LocalStorage;
3608 }
3609 } else {
3610 // Non-descriptor by-ref: the pointer already references data
3611 // directly. Shuffle into the destination alloca.
3612 ShuffleDestAddr = DestElementAddr;
3613 }
3614 }
3615
3616 shuffleAndStore(AllocaIP, SrcAddr: ShuffleSrcAddr, DstAddr: ShuffleDestAddr, ElemType: ShuffleType,
3617 Offset: RemoteLaneOffset, ReductionArrayTy, IsByRefElem);
3618
3619 if (IsByRefElem && RI.DataPtrPtrGen) {
3620 // Copy descriptor from source and update base_ptr to shuffled data
3621 Value *DestDescriptorAddr = Builder.CreatePointerBitCastOrAddrSpaceCast(
3622 V: DestAlloca, DestTy: Builder.getPtrTy(), Name: ".ascast");
3623
3624 InsertPointOrErrorTy GenResult = generateReductionDescriptor(
3625 DescriptorAddr: DestDescriptorAddr, DataPtr: LocalStorage, SrcDescriptorAddr: SrcElementAddr,
3626 DescriptorType: RI.ByRefAllocatedType, DataPtrPtrGen: RI.DataPtrPtrGen);
3627
3628 if (!GenResult)
3629 return GenResult.takeError();
3630 }
3631 } else {
3632 switch (RI.EvaluationKind) {
3633 case EvalKind::Scalar: {
3634 Value *Elem = Builder.CreateLoad(Ty: RI.ElementType, Ptr: SrcElementAddr);
3635 // Store the source element value to the dest element address.
3636 Builder.CreateStore(Val: Elem, Ptr: DestElementAddr);
3637 break;
3638 }
3639 case EvalKind::Complex: {
3640 Value *SrcRealPtr = Builder.CreateConstInBoundsGEP2_32(
3641 Ty: RI.ElementType, Ptr: SrcElementAddr, Idx0: 0, Idx1: 0, Name: ".realp");
3642 Value *SrcReal = Builder.CreateLoad(
3643 Ty: RI.ElementType->getStructElementType(N: 0), Ptr: SrcRealPtr, Name: ".real");
3644 Value *SrcImgPtr = Builder.CreateConstInBoundsGEP2_32(
3645 Ty: RI.ElementType, Ptr: SrcElementAddr, Idx0: 0, Idx1: 1, Name: ".imagp");
3646 Value *SrcImg = Builder.CreateLoad(
3647 Ty: RI.ElementType->getStructElementType(N: 1), Ptr: SrcImgPtr, Name: ".imag");
3648
3649 Value *DestRealPtr = Builder.CreateConstInBoundsGEP2_32(
3650 Ty: RI.ElementType, Ptr: DestElementAddr, Idx0: 0, Idx1: 0, Name: ".realp");
3651 Value *DestImgPtr = Builder.CreateConstInBoundsGEP2_32(
3652 Ty: RI.ElementType, Ptr: DestElementAddr, Idx0: 0, Idx1: 1, Name: ".imagp");
3653 Builder.CreateStore(Val: SrcReal, Ptr: DestRealPtr);
3654 Builder.CreateStore(Val: SrcImg, Ptr: DestImgPtr);
3655 break;
3656 }
3657 case EvalKind::Aggregate: {
3658 Value *SizeVal = Builder.getInt64(
3659 C: M.getDataLayout().getTypeStoreSize(Ty: RI.ElementType));
3660 Builder.CreateMemCpy(
3661 Dst: DestElementAddr, DstAlign: M.getDataLayout().getPrefTypeAlign(Ty: RI.ElementType),
3662 Src: SrcElementAddr, SrcAlign: M.getDataLayout().getPrefTypeAlign(Ty: RI.ElementType),
3663 Size: SizeVal, isVolatile: false);
3664 break;
3665 }
3666 };
3667 }
3668
3669 // Step 3.1: Modify reference in dest Reduce list as needed.
3670 // Modifying the reference in Reduce list to point to the newly
3671 // created element. The element is live in the current function
3672 // scope and that of functions it invokes (i.e., reduce_function).
3673 // RemoteReduceData[i] = (void*)&RemoteElem
3674 if (UpdateDestListPtr) {
3675 Value *CastDestAddr = Builder.CreatePointerBitCastOrAddrSpaceCast(
3676 V: DestElementAddr, DestTy: Builder.getPtrTy(),
3677 Name: DestElementAddr->getName() + ".ascast");
3678 Builder.CreateStore(Val: CastDestAddr, Ptr: DestElementPtrAddr);
3679 }
3680 }
3681
3682 return Error::success();
3683}
3684
3685Expected<Function *> OpenMPIRBuilder::emitInterWarpCopyFunction(
3686 const LocationDescription &Loc, ArrayRef<ReductionInfo> ReductionInfos,
3687 AttributeList FuncAttrs, ArrayRef<bool> IsByRef) {
3688 IRBuilder<>::InsertPointGuard IPG(Builder);
3689 LLVMContext &Ctx = M.getContext();
3690 FunctionType *FuncTy = FunctionType::get(
3691 Result: Builder.getVoidTy(), Params: {Builder.getPtrTy(), Builder.getInt32Ty()},
3692 /* IsVarArg */ isVarArg: false);
3693 Function *WcFunc =
3694 Function::Create(Ty: FuncTy, Linkage: GlobalVariable::InternalLinkage,
3695 N: "_omp_reduction_inter_warp_copy_func", M: &M);
3696 WcFunc->setCallingConv(Config.getRuntimeCC());
3697 WcFunc->setAttributes(FuncAttrs);
3698 WcFunc->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
3699 WcFunc->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
3700 BasicBlock *EntryBB = BasicBlock::Create(Context&: M.getContext(), Name: "entry", Parent: WcFunc);
3701 Builder.SetInsertPoint(EntryBB);
3702 Builder.SetCurrentDebugLocation(llvm::DebugLoc());
3703
3704 // ReduceList: thread local Reduce list.
3705 // At the stage of the computation when this function is called, partially
3706 // aggregated values reside in the first lane of every active warp.
3707 Argument *ReduceListArg = WcFunc->getArg(i: 0);
3708 // NumWarps: number of warps active in the parallel region. This could
3709 // be smaller than 32 (max warps in a CTA) for partial block reduction.
3710 Argument *NumWarpsArg = WcFunc->getArg(i: 1);
3711
3712 // This array is used as a medium to transfer, one reduce element at a time,
3713 // the data from the first lane of every warp to lanes in the first warp
3714 // in order to perform the final step of a reduction in a parallel region
3715 // (reduction across warps). The array is placed in NVPTX __shared__ memory
3716 // for reduced latency, as well as to have a distinct copy for concurrently
3717 // executing target regions. The array is declared with common linkage so
3718 // as to be shared across compilation units.
3719 StringRef TransferMediumName =
3720 "__openmp_nvptx_data_transfer_temporary_storage";
3721 GlobalVariable *TransferMedium = M.getGlobalVariable(Name: TransferMediumName);
3722 unsigned WarpSize = Config.getGridValue().GV_Warp_Size;
3723 ArrayType *ArrayTy = ArrayType::get(ElementType: Builder.getInt32Ty(), NumElements: WarpSize);
3724 if (!TransferMedium) {
3725 TransferMedium = new GlobalVariable(
3726 M, ArrayTy, /*isConstant=*/false, GlobalVariable::WeakAnyLinkage,
3727 UndefValue::get(T: ArrayTy), TransferMediumName,
3728 /*InsertBefore=*/nullptr, GlobalVariable::NotThreadLocal,
3729 /*AddressSpace=*/3);
3730 }
3731
3732 // Get the CUDA thread id of the current OpenMP thread on the GPU.
3733 Value *GPUThreadID = getGPUThreadID();
3734 // nvptx_lane_id = nvptx_id % warpsize
3735 Value *LaneID = getNVPTXLaneID();
3736 // nvptx_warp_id = nvptx_id / warpsize
3737 Value *WarpID = getNVPTXWarpID();
3738
3739 InsertPointTy AllocaIP = Builder.GetInsertBlock()->getFirstInsertionPt();
3740 Type *Arg0Type = ReduceListArg->getType();
3741 Type *Arg1Type = NumWarpsArg->getType();
3742 Builder.restoreIP(IP: AllocaIP);
3743 AllocaInst *ReduceListAlloca = Builder.CreateAlloca(
3744 Ty: Arg0Type, ArraySize: nullptr, Name: ReduceListArg->getName() + ".addr");
3745 AllocaInst *NumWarpsAlloca =
3746 Builder.CreateAlloca(Ty: Arg1Type, ArraySize: nullptr, Name: NumWarpsArg->getName() + ".addr");
3747 Value *ReduceListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
3748 V: ReduceListAlloca, DestTy: Arg0Type, Name: ReduceListAlloca->getName() + ".ascast");
3749 Value *NumWarpsAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
3750 V: NumWarpsAlloca, DestTy: Builder.getPtrTy(AddrSpace: 0),
3751 Name: NumWarpsAlloca->getName() + ".ascast");
3752 Builder.CreateStore(Val: ReduceListArg, Ptr: ReduceListAddrCast);
3753 Builder.CreateStore(Val: NumWarpsArg, Ptr: NumWarpsAddrCast);
3754 AllocaIP = getInsertPointAfterInstr(I: NumWarpsAlloca);
3755 InsertPointTy CodeGenIP =
3756 getInsertPointAfterInstr(I: &Builder.GetInsertBlock()->back());
3757 Builder.restoreIP(IP: CodeGenIP);
3758
3759 Value *ReduceList =
3760 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ReduceListAddrCast);
3761
3762 for (auto En : enumerate(First&: ReductionInfos)) {
3763 //
3764 // Warp master copies reduce element to transfer medium in __shared__
3765 // memory.
3766 //
3767 const ReductionInfo &RI = En.value();
3768 bool IsByRefElem = !IsByRef.empty() && IsByRef[En.index()];
3769 unsigned RealTySize = M.getDataLayout().getTypeAllocSize(
3770 Ty: IsByRefElem ? RI.ByRefElementType : RI.ElementType);
3771 for (unsigned TySize = 4; TySize > 0 && RealTySize > 0; TySize /= 2) {
3772 Type *CType = Builder.getIntNTy(N: TySize * 8);
3773
3774 unsigned NumIters = RealTySize / TySize;
3775 if (NumIters == 0)
3776 continue;
3777 Value *Cnt = nullptr;
3778 Value *CntAddr = nullptr;
3779 BasicBlock *PrecondBB = nullptr;
3780 BasicBlock *ExitBB = nullptr;
3781 if (NumIters > 1) {
3782 CodeGenIP = Builder.saveIP();
3783 Builder.restoreIP(IP: AllocaIP);
3784 CntAddr =
3785 Builder.CreateAlloca(Ty: Builder.getInt32Ty(), ArraySize: nullptr, Name: ".cnt.addr");
3786
3787 CntAddr = Builder.CreateAddrSpaceCast(V: CntAddr, DestTy: Builder.getPtrTy(),
3788 Name: CntAddr->getName() + ".ascast");
3789 Builder.restoreIP(IP: CodeGenIP);
3790 Builder.CreateStore(Val: Constant::getNullValue(Ty: Builder.getInt32Ty()),
3791 Ptr: CntAddr,
3792 /*Volatile=*/isVolatile: false);
3793 PrecondBB = BasicBlock::Create(Context&: Ctx, Name: "precond");
3794 ExitBB = BasicBlock::Create(Context&: Ctx, Name: "exit");
3795 BasicBlock *BodyBB = BasicBlock::Create(Context&: Ctx, Name: "body");
3796 emitBlock(BB: PrecondBB, CurFn: Builder.GetInsertBlock()->getParent());
3797 Cnt = Builder.CreateLoad(Ty: Builder.getInt32Ty(), Ptr: CntAddr,
3798 /*Volatile=*/isVolatile: false);
3799 Value *Cmp = Builder.CreateICmpULT(
3800 LHS: Cnt, RHS: ConstantInt::get(Ty: Builder.getInt32Ty(), V: NumIters));
3801 Builder.CreateCondBr(Cond: Cmp, True: BodyBB, False: ExitBB);
3802 emitBlock(BB: BodyBB, CurFn: Builder.GetInsertBlock()->getParent());
3803 }
3804
3805 // kmpc_barrier.
3806 InsertPointOrErrorTy BarrierIP1 =
3807 createBarrier(Loc: LocationDescription(Builder.saveIP(), DebugLoc()),
3808 Kind: omp::Directive::OMPD_unknown,
3809 /* ForceSimpleCall */ false,
3810 /* CheckCancelFlag */ true);
3811 if (!BarrierIP1)
3812 return BarrierIP1.takeError();
3813 BasicBlock *ThenBB = BasicBlock::Create(Context&: Ctx, Name: "then");
3814 BasicBlock *ElseBB = BasicBlock::Create(Context&: Ctx, Name: "else");
3815 BasicBlock *MergeBB = BasicBlock::Create(Context&: Ctx, Name: "ifcont");
3816
3817 // if (lane_id == 0)
3818 Value *IsWarpMaster = Builder.CreateIsNull(Arg: LaneID, Name: "warp_master");
3819 Builder.CreateCondBr(Cond: IsWarpMaster, True: ThenBB, False: ElseBB);
3820 emitBlock(BB: ThenBB, CurFn: Builder.GetInsertBlock()->getParent());
3821
3822 // Reduce element = LocalReduceList[i]
3823 auto *RedListArrayTy =
3824 ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: ReductionInfos.size());
3825 Type *IndexTy = Builder.getIndexTy(
3826 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
3827 Value *ElemPtrPtr =
3828 Builder.CreateInBoundsGEP(Ty: RedListArrayTy, Ptr: ReduceList,
3829 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0),
3830 ConstantInt::get(Ty: IndexTy, V: En.index())});
3831 // elemptr = ((CopyType*)(elemptrptr)) + I
3832 Value *ElemPtr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ElemPtrPtr);
3833
3834 if (IsByRefElem && RI.DataPtrPtrGen) {
3835 InsertPointOrErrorTy GenRes =
3836 RI.DataPtrPtrGen(Builder.saveIP(), ElemPtr, ElemPtr);
3837
3838 if (!GenRes)
3839 return GenRes.takeError();
3840
3841 ElemPtr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ElemPtr);
3842 }
3843
3844 if (NumIters > 1)
3845 ElemPtr = Builder.CreateGEP(Ty: Builder.getInt32Ty(), Ptr: ElemPtr, IdxList: Cnt);
3846
3847 // Get pointer to location in transfer medium.
3848 // MediumPtr = &medium[warp_id]
3849 Value *MediumPtr = Builder.CreateInBoundsGEP(
3850 Ty: ArrayTy, Ptr: TransferMedium, IdxList: {Builder.getInt64(C: 0), WarpID});
3851 // elem = *elemptr
3852 //*MediumPtr = elem
3853 Value *Elem = Builder.CreateLoad(Ty: CType, Ptr: ElemPtr);
3854 // Store the source element value to the dest element address.
3855 Builder.CreateStore(Val: Elem, Ptr: MediumPtr,
3856 /*IsVolatile*/ isVolatile: true);
3857 Builder.CreateBr(Dest: MergeBB);
3858
3859 // else
3860 emitBlock(BB: ElseBB, CurFn: Builder.GetInsertBlock()->getParent());
3861 Builder.CreateBr(Dest: MergeBB);
3862
3863 // endif
3864 emitBlock(BB: MergeBB, CurFn: Builder.GetInsertBlock()->getParent());
3865 InsertPointOrErrorTy BarrierIP2 =
3866 createBarrier(Loc: LocationDescription(Builder.saveIP(), DebugLoc()),
3867 Kind: omp::Directive::OMPD_unknown,
3868 /* ForceSimpleCall */ false,
3869 /* CheckCancelFlag */ true);
3870 if (!BarrierIP2)
3871 return BarrierIP2.takeError();
3872
3873 // Warp 0 copies reduce element from transfer medium
3874 BasicBlock *W0ThenBB = BasicBlock::Create(Context&: Ctx, Name: "then");
3875 BasicBlock *W0ElseBB = BasicBlock::Create(Context&: Ctx, Name: "else");
3876 BasicBlock *W0MergeBB = BasicBlock::Create(Context&: Ctx, Name: "ifcont");
3877
3878 Value *NumWarpsVal =
3879 Builder.CreateLoad(Ty: Builder.getInt32Ty(), Ptr: NumWarpsAddrCast);
3880 // Up to 32 threads in warp 0 are active.
3881 Value *IsActiveThread =
3882 Builder.CreateICmpULT(LHS: GPUThreadID, RHS: NumWarpsVal, Name: "is_active_thread");
3883 Builder.CreateCondBr(Cond: IsActiveThread, True: W0ThenBB, False: W0ElseBB);
3884
3885 emitBlock(BB: W0ThenBB, CurFn: Builder.GetInsertBlock()->getParent());
3886
3887 // SecMediumPtr = &medium[tid]
3888 // SrcMediumVal = *SrcMediumPtr
3889 Value *SrcMediumPtrVal = Builder.CreateInBoundsGEP(
3890 Ty: ArrayTy, Ptr: TransferMedium, IdxList: {Builder.getInt64(C: 0), GPUThreadID});
3891 // TargetElemPtr = (CopyType*)(SrcDataAddr[i]) + I
3892 Value *TargetElemPtrPtr =
3893 Builder.CreateInBoundsGEP(Ty: RedListArrayTy, Ptr: ReduceList,
3894 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0),
3895 ConstantInt::get(Ty: IndexTy, V: En.index())});
3896 Value *TargetElemPtrVal =
3897 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: TargetElemPtrPtr);
3898 Value *TargetElemPtr = TargetElemPtrVal;
3899
3900 if (IsByRefElem && RI.DataPtrPtrGen) {
3901 InsertPointOrErrorTy GenRes =
3902 RI.DataPtrPtrGen(Builder.saveIP(), TargetElemPtr, TargetElemPtr);
3903
3904 if (!GenRes)
3905 return GenRes.takeError();
3906
3907 TargetElemPtr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: TargetElemPtr);
3908 }
3909
3910 if (NumIters > 1)
3911 TargetElemPtr =
3912 Builder.CreateGEP(Ty: Builder.getInt32Ty(), Ptr: TargetElemPtr, IdxList: Cnt);
3913
3914 // *TargetElemPtr = SrcMediumVal;
3915 Value *SrcMediumValue =
3916 Builder.CreateLoad(Ty: CType, Ptr: SrcMediumPtrVal, /*IsVolatile*/ isVolatile: true);
3917 Builder.CreateStore(Val: SrcMediumValue, Ptr: TargetElemPtr);
3918 Builder.CreateBr(Dest: W0MergeBB);
3919
3920 emitBlock(BB: W0ElseBB, CurFn: Builder.GetInsertBlock()->getParent());
3921 Builder.CreateBr(Dest: W0MergeBB);
3922
3923 emitBlock(BB: W0MergeBB, CurFn: Builder.GetInsertBlock()->getParent());
3924
3925 if (NumIters > 1) {
3926 Cnt = Builder.CreateNSWAdd(
3927 LHS: Cnt, RHS: ConstantInt::get(Ty: Builder.getInt32Ty(), /*V=*/1));
3928 Builder.CreateStore(Val: Cnt, Ptr: CntAddr, /*Volatile=*/isVolatile: false);
3929
3930 auto *CurFn = Builder.GetInsertBlock()->getParent();
3931 emitBranch(Target: PrecondBB);
3932 emitBlock(BB: ExitBB, CurFn);
3933 }
3934 RealTySize %= TySize;
3935 }
3936 }
3937
3938 Builder.CreateRetVoid();
3939
3940 return WcFunc;
3941}
3942
3943Expected<Function *> OpenMPIRBuilder::emitShuffleAndReduceFunction(
3944 ArrayRef<ReductionInfo> ReductionInfos, Function *ReduceFn,
3945 AttributeList FuncAttrs, ArrayRef<bool> IsByRef) {
3946 LLVMContext &Ctx = M.getContext();
3947 IRBuilder<>::InsertPointGuard IPG(Builder);
3948 FunctionType *FuncTy =
3949 FunctionType::get(Result: Builder.getVoidTy(),
3950 Params: {Builder.getPtrTy(), Builder.getInt16Ty(),
3951 Builder.getInt16Ty(), Builder.getInt16Ty()},
3952 /* IsVarArg */ isVarArg: false);
3953 Function *SarFunc =
3954 Function::Create(Ty: FuncTy, Linkage: GlobalVariable::InternalLinkage,
3955 N: "_omp_reduction_shuffle_and_reduce_func", M: &M);
3956 SarFunc->setCallingConv(Config.getRuntimeCC());
3957 SarFunc->setAttributes(FuncAttrs);
3958 SarFunc->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
3959 SarFunc->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
3960 SarFunc->addParamAttr(ArgNo: 2, Kind: Attribute::NoUndef);
3961 SarFunc->addParamAttr(ArgNo: 3, Kind: Attribute::NoUndef);
3962 SarFunc->addParamAttr(ArgNo: 1, Kind: Attribute::SExt);
3963 SarFunc->addParamAttr(ArgNo: 2, Kind: Attribute::SExt);
3964 SarFunc->addParamAttr(ArgNo: 3, Kind: Attribute::SExt);
3965 BasicBlock *EntryBB = BasicBlock::Create(Context&: M.getContext(), Name: "entry", Parent: SarFunc);
3966 Builder.SetInsertPoint(EntryBB);
3967 Builder.SetCurrentDebugLocation(llvm::DebugLoc());
3968
3969 // Thread local Reduce list used to host the values of data to be reduced.
3970 Argument *ReduceListArg = SarFunc->getArg(i: 0);
3971 // Current lane id; could be logical.
3972 Argument *LaneIDArg = SarFunc->getArg(i: 1);
3973 // Offset of the remote source lane relative to the current lane.
3974 Argument *RemoteLaneOffsetArg = SarFunc->getArg(i: 2);
3975 // Algorithm version. This is expected to be known at compile time.
3976 Argument *AlgoVerArg = SarFunc->getArg(i: 3);
3977
3978 Type *ReduceListArgType = ReduceListArg->getType();
3979 Type *LaneIDArgType = LaneIDArg->getType();
3980 Type *LaneIDArgPtrType = Builder.getPtrTy(AddrSpace: 0);
3981 Value *ReduceListAlloca = Builder.CreateAlloca(
3982 Ty: ReduceListArgType, ArraySize: nullptr, Name: ReduceListArg->getName() + ".addr");
3983 Value *LaneIdAlloca = Builder.CreateAlloca(Ty: LaneIDArgType, ArraySize: nullptr,
3984 Name: LaneIDArg->getName() + ".addr");
3985 Value *RemoteLaneOffsetAlloca = Builder.CreateAlloca(
3986 Ty: LaneIDArgType, ArraySize: nullptr, Name: RemoteLaneOffsetArg->getName() + ".addr");
3987 Value *AlgoVerAlloca = Builder.CreateAlloca(Ty: LaneIDArgType, ArraySize: nullptr,
3988 Name: AlgoVerArg->getName() + ".addr");
3989 ArrayType *RedListArrayTy =
3990 ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: ReductionInfos.size());
3991
3992 // Create a local thread-private variable to host the Reduce list
3993 // from a remote lane.
3994 Instruction *RemoteReductionListAlloca = Builder.CreateAlloca(
3995 Ty: RedListArrayTy, ArraySize: nullptr, Name: ".omp.reduction.remote_reduce_list");
3996
3997 Value *ReduceListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
3998 V: ReduceListAlloca, DestTy: ReduceListArgType,
3999 Name: ReduceListAlloca->getName() + ".ascast");
4000 Value *LaneIdAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4001 V: LaneIdAlloca, DestTy: LaneIDArgPtrType, Name: LaneIdAlloca->getName() + ".ascast");
4002 Value *RemoteLaneOffsetAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4003 V: RemoteLaneOffsetAlloca, DestTy: LaneIDArgPtrType,
4004 Name: RemoteLaneOffsetAlloca->getName() + ".ascast");
4005 Value *AlgoVerAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4006 V: AlgoVerAlloca, DestTy: LaneIDArgPtrType, Name: AlgoVerAlloca->getName() + ".ascast");
4007 Value *RemoteListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4008 V: RemoteReductionListAlloca, DestTy: Builder.getPtrTy(),
4009 Name: RemoteReductionListAlloca->getName() + ".ascast");
4010
4011 Builder.CreateStore(Val: ReduceListArg, Ptr: ReduceListAddrCast);
4012 Builder.CreateStore(Val: LaneIDArg, Ptr: LaneIdAddrCast);
4013 Builder.CreateStore(Val: RemoteLaneOffsetArg, Ptr: RemoteLaneOffsetAddrCast);
4014 Builder.CreateStore(Val: AlgoVerArg, Ptr: AlgoVerAddrCast);
4015
4016 Value *ReduceList = Builder.CreateLoad(Ty: ReduceListArgType, Ptr: ReduceListAddrCast);
4017 Value *LaneId = Builder.CreateLoad(Ty: LaneIDArgType, Ptr: LaneIdAddrCast);
4018 Value *RemoteLaneOffset =
4019 Builder.CreateLoad(Ty: LaneIDArgType, Ptr: RemoteLaneOffsetAddrCast);
4020 Value *AlgoVer = Builder.CreateLoad(Ty: LaneIDArgType, Ptr: AlgoVerAddrCast);
4021
4022 InsertPointTy AllocaIP = getInsertPointAfterInstr(I: RemoteReductionListAlloca);
4023
4024 // This loop iterates through the list of reduce elements and copies,
4025 // element by element, from a remote lane in the warp to RemoteReduceList,
4026 // hosted on the thread's stack.
4027 Error EmitRedLsCpRes = emitReductionListCopy(
4028 AllocaIP, Action: CopyAction::RemoteLaneToThread, ReductionArrayTy: RedListArrayTy, ReductionInfos,
4029 SrcBase: ReduceList, DestBase: RemoteListAddrCast, IsByRef,
4030 CopyOptions: {.RemoteLaneOffset: RemoteLaneOffset, .ScratchpadIndex: nullptr, .ScratchpadWidth: nullptr});
4031
4032 if (EmitRedLsCpRes)
4033 return EmitRedLsCpRes;
4034
4035 // The actions to be performed on the Remote Reduce list is dependent
4036 // on the algorithm version.
4037 //
4038 // if (AlgoVer==0) || (AlgoVer==1 && (LaneId < Offset)) || (AlgoVer==2 &&
4039 // LaneId % 2 == 0 && Offset > 0):
4040 // do the reduction value aggregation
4041 //
4042 // The thread local variable Reduce list is mutated in place to host the
4043 // reduced data, which is the aggregated value produced from local and
4044 // remote lanes.
4045 //
4046 // Note that AlgoVer is expected to be a constant integer known at compile
4047 // time.
4048 // When AlgoVer==0, the first conjunction evaluates to true, making
4049 // the entire predicate true during compile time.
4050 // When AlgoVer==1, the second conjunction has only the second part to be
4051 // evaluated during runtime. Other conjunctions evaluates to false
4052 // during compile time.
4053 // When AlgoVer==2, the third conjunction has only the second part to be
4054 // evaluated during runtime. Other conjunctions evaluates to false
4055 // during compile time.
4056 Value *CondAlgo0 = Builder.CreateIsNull(Arg: AlgoVer);
4057 Value *Algo1 = Builder.CreateICmpEQ(LHS: AlgoVer, RHS: Builder.getInt16(C: 1));
4058 Value *LaneComp = Builder.CreateICmpULT(LHS: LaneId, RHS: RemoteLaneOffset);
4059 Value *CondAlgo1 = Builder.CreateAnd(LHS: Algo1, RHS: LaneComp);
4060 Value *Algo2 = Builder.CreateICmpEQ(LHS: AlgoVer, RHS: Builder.getInt16(C: 2));
4061 Value *LaneIdAnd1 = Builder.CreateAnd(LHS: LaneId, RHS: Builder.getInt16(C: 1));
4062 Value *LaneIdComp = Builder.CreateIsNull(Arg: LaneIdAnd1);
4063 Value *Algo2AndLaneIdComp = Builder.CreateAnd(LHS: Algo2, RHS: LaneIdComp);
4064 Value *RemoteOffsetComp =
4065 Builder.CreateICmpSGT(LHS: RemoteLaneOffset, RHS: Builder.getInt16(C: 0));
4066 Value *CondAlgo2 = Builder.CreateAnd(LHS: Algo2AndLaneIdComp, RHS: RemoteOffsetComp);
4067 Value *CA0OrCA1 = Builder.CreateOr(LHS: CondAlgo0, RHS: CondAlgo1);
4068 Value *CondReduce = Builder.CreateOr(LHS: CA0OrCA1, RHS: CondAlgo2);
4069
4070 BasicBlock *ThenBB = BasicBlock::Create(Context&: Ctx, Name: "then");
4071 BasicBlock *ElseBB = BasicBlock::Create(Context&: Ctx, Name: "else");
4072 BasicBlock *MergeBB = BasicBlock::Create(Context&: Ctx, Name: "ifcont");
4073
4074 Builder.CreateCondBr(Cond: CondReduce, True: ThenBB, False: ElseBB);
4075 emitBlock(BB: ThenBB, CurFn: Builder.GetInsertBlock()->getParent());
4076 Value *LocalReduceListPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
4077 V: ReduceList, DestTy: Builder.getPtrTy());
4078 Value *RemoteReduceListPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
4079 V: RemoteListAddrCast, DestTy: Builder.getPtrTy());
4080 createRuntimeFunctionCall(Callee: ReduceFn, Args: {LocalReduceListPtr, RemoteReduceListPtr})
4081 ->addFnAttr(Kind: Attribute::NoUnwind);
4082 Builder.CreateBr(Dest: MergeBB);
4083
4084 emitBlock(BB: ElseBB, CurFn: Builder.GetInsertBlock()->getParent());
4085 Builder.CreateBr(Dest: MergeBB);
4086
4087 emitBlock(BB: MergeBB, CurFn: Builder.GetInsertBlock()->getParent());
4088
4089 // if (AlgoVer==1 && (LaneId >= Offset)) copy Remote Reduce list to local
4090 // Reduce list.
4091 Algo1 = Builder.CreateICmpEQ(LHS: AlgoVer, RHS: Builder.getInt16(C: 1));
4092 Value *LaneIdGtOffset = Builder.CreateICmpUGE(LHS: LaneId, RHS: RemoteLaneOffset);
4093 Value *CondCopy = Builder.CreateAnd(LHS: Algo1, RHS: LaneIdGtOffset);
4094
4095 BasicBlock *CpyThenBB = BasicBlock::Create(Context&: Ctx, Name: "then");
4096 BasicBlock *CpyElseBB = BasicBlock::Create(Context&: Ctx, Name: "else");
4097 BasicBlock *CpyMergeBB = BasicBlock::Create(Context&: Ctx, Name: "ifcont");
4098 Builder.CreateCondBr(Cond: CondCopy, True: CpyThenBB, False: CpyElseBB);
4099
4100 emitBlock(BB: CpyThenBB, CurFn: Builder.GetInsertBlock()->getParent());
4101
4102 EmitRedLsCpRes = emitReductionListCopy(
4103 AllocaIP, Action: CopyAction::ThreadCopy, ReductionArrayTy: RedListArrayTy, ReductionInfos,
4104 SrcBase: RemoteListAddrCast, DestBase: ReduceList, IsByRef);
4105
4106 if (EmitRedLsCpRes)
4107 return EmitRedLsCpRes;
4108
4109 Builder.CreateBr(Dest: CpyMergeBB);
4110
4111 emitBlock(BB: CpyElseBB, CurFn: Builder.GetInsertBlock()->getParent());
4112 Builder.CreateBr(Dest: CpyMergeBB);
4113
4114 emitBlock(BB: CpyMergeBB, CurFn: Builder.GetInsertBlock()->getParent());
4115
4116 Builder.CreateRetVoid();
4117
4118 return SarFunc;
4119}
4120
4121OpenMPIRBuilder::InsertPointOrErrorTy
4122OpenMPIRBuilder::generateReductionDescriptor(
4123 Value *DescriptorAddr, Value *DataPtr, Value *SrcDescriptorAddr,
4124 Type *DescriptorType,
4125 function_ref<InsertPointOrErrorTy(InsertPointTy, Value *, Value *&)>
4126 DataPtrPtrGen) {
4127
4128 // Copy the source descriptor to preserve all metadata (rank, extents,
4129 // strides, etc.)
4130 Value *DescriptorSize =
4131 Builder.getInt64(C: M.getDataLayout().getTypeStoreSize(Ty: DescriptorType));
4132 Builder.CreateMemCpy(
4133 Dst: DescriptorAddr, DstAlign: M.getDataLayout().getPrefTypeAlign(Ty: DescriptorType),
4134 Src: SrcDescriptorAddr, SrcAlign: M.getDataLayout().getPrefTypeAlign(Ty: DescriptorType),
4135 Size: DescriptorSize);
4136
4137 // Update the base pointer field to point to the local shuffled data
4138 Value *DataPtrField;
4139 InsertPointOrErrorTy GenResult =
4140 DataPtrPtrGen(Builder.saveIP(), DescriptorAddr, DataPtrField);
4141
4142 if (!GenResult)
4143 return GenResult.takeError();
4144
4145 Builder.CreateStore(Val: Builder.CreatePointerBitCastOrAddrSpaceCast(
4146 V: DataPtr, DestTy: Builder.getPtrTy(), Name: ".ascast"),
4147 Ptr: DataPtrField);
4148
4149 return Builder.saveIP();
4150}
4151
4152Expected<Value *> OpenMPIRBuilder::createReductionDescriptorCopy(
4153 InsertPointTy AllocaIP, const ReductionInfo &RI, Value *DataPtr,
4154 Value *SrcDescriptorAddr, Type *DescriptorPtrTy, const Twine &Name) {
4155 InsertPointTy OldIP = Builder.saveIP();
4156 Builder.restoreIP(IP: AllocaIP);
4157
4158 AllocaInst *DescriptorAlloca =
4159 Builder.CreateAlloca(Ty: RI.ByRefAllocatedType, ArraySize: nullptr, Name);
4160 DescriptorAlloca->setAlignment(
4161 M.getDataLayout().getPrefTypeAlign(Ty: RI.ByRefAllocatedType));
4162 Value *DescriptorAddr = Builder.CreatePointerBitCastOrAddrSpaceCast(
4163 V: DescriptorAlloca, DestTy: DescriptorPtrTy,
4164 Name: DescriptorAlloca->getName() + ".ascast");
4165
4166 Builder.restoreIP(IP: OldIP);
4167
4168 InsertPointOrErrorTy GenResult =
4169 generateReductionDescriptor(DescriptorAddr, DataPtr, SrcDescriptorAddr,
4170 DescriptorType: RI.ByRefAllocatedType, DataPtrPtrGen: RI.DataPtrPtrGen);
4171 if (!GenResult)
4172 return GenResult.takeError();
4173
4174 return DescriptorAddr;
4175}
4176
4177Expected<Function *> OpenMPIRBuilder::emitListToGlobalCopyFunction(
4178 ArrayRef<ReductionInfo> ReductionInfos, Type *ReductionsBufferTy,
4179 AttributeList FuncAttrs, ArrayRef<bool> IsByRef) {
4180 IRBuilder<>::InsertPointGuard IPG(Builder);
4181 LLVMContext &Ctx = M.getContext();
4182 FunctionType *FuncTy = FunctionType::get(
4183 Result: Builder.getVoidTy(),
4184 Params: {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},
4185 /* IsVarArg */ isVarArg: false);
4186 Function *LtGCFunc =
4187 Function::Create(Ty: FuncTy, Linkage: GlobalVariable::InternalLinkage,
4188 N: "_omp_reduction_list_to_global_copy_func", M: &M);
4189 LtGCFunc->setAttributes(FuncAttrs);
4190 LtGCFunc->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
4191 LtGCFunc->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
4192 LtGCFunc->addParamAttr(ArgNo: 2, Kind: Attribute::NoUndef);
4193
4194 BasicBlock *EntryBlock = BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: LtGCFunc);
4195 Builder.SetInsertPoint(EntryBlock);
4196 Builder.SetCurrentDebugLocation(llvm::DebugLoc());
4197
4198 // Buffer: global reduction buffer.
4199 Argument *BufferArg = LtGCFunc->getArg(i: 0);
4200 // Idx: index of the buffer.
4201 Argument *IdxArg = LtGCFunc->getArg(i: 1);
4202 // ReduceList: thread local Reduce list.
4203 Argument *ReduceListArg = LtGCFunc->getArg(i: 2);
4204
4205 Value *BufferArgAlloca = Builder.CreateAlloca(Ty: Builder.getPtrTy(), ArraySize: nullptr,
4206 Name: BufferArg->getName() + ".addr");
4207 Value *IdxArgAlloca = Builder.CreateAlloca(Ty: Builder.getInt32Ty(), ArraySize: nullptr,
4208 Name: IdxArg->getName() + ".addr");
4209 Value *ReduceListArgAlloca = Builder.CreateAlloca(
4210 Ty: Builder.getPtrTy(), ArraySize: nullptr, Name: ReduceListArg->getName() + ".addr");
4211 Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4212 V: BufferArgAlloca, DestTy: Builder.getPtrTy(),
4213 Name: BufferArgAlloca->getName() + ".ascast");
4214 Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4215 V: IdxArgAlloca, DestTy: Builder.getPtrTy(), Name: IdxArgAlloca->getName() + ".ascast");
4216 Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4217 V: ReduceListArgAlloca, DestTy: Builder.getPtrTy(),
4218 Name: ReduceListArgAlloca->getName() + ".ascast");
4219
4220 Builder.CreateStore(Val: BufferArg, Ptr: BufferArgAddrCast);
4221 Builder.CreateStore(Val: IdxArg, Ptr: IdxArgAddrCast);
4222 Builder.CreateStore(Val: ReduceListArg, Ptr: ReduceListArgAddrCast);
4223
4224 Value *LocalReduceList =
4225 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ReduceListArgAddrCast);
4226 Value *BufferArgVal =
4227 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: BufferArgAddrCast);
4228 Value *Idxs[] = {Builder.CreateLoad(Ty: Builder.getInt32Ty(), Ptr: IdxArgAddrCast)};
4229 Type *IndexTy = Builder.getIndexTy(
4230 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
4231 for (auto En : enumerate(First&: ReductionInfos)) {
4232 const ReductionInfo &RI = En.value();
4233 auto *RedListArrayTy =
4234 ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: ReductionInfos.size());
4235 // Reduce element = LocalReduceList[i]
4236 Value *ElemPtrPtr = Builder.CreateInBoundsGEP(
4237 Ty: RedListArrayTy, Ptr: LocalReduceList,
4238 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
4239 // elemptr = ((CopyType*)(elemptrptr)) + I
4240 Value *ElemPtr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ElemPtrPtr);
4241
4242 // Global = Buffer.VD[Idx];
4243 Value *BufferVD =
4244 Builder.CreateInBoundsGEP(Ty: ReductionsBufferTy, Ptr: BufferArgVal, IdxList: Idxs);
4245 Value *GlobVal = Builder.CreateConstInBoundsGEP2_32(
4246 Ty: ReductionsBufferTy, Ptr: BufferVD, Idx0: 0, Idx1: En.index());
4247
4248 switch (RI.EvaluationKind) {
4249 case EvalKind::Scalar: {
4250 Value *TargetElement;
4251
4252 if (IsByRef.empty() || !IsByRef[En.index()]) {
4253 TargetElement = Builder.CreateLoad(Ty: RI.ElementType, Ptr: ElemPtr);
4254 } else {
4255 if (RI.DataPtrPtrGen) {
4256 InsertPointOrErrorTy GenResult =
4257 RI.DataPtrPtrGen(Builder.saveIP(), ElemPtr, ElemPtr);
4258
4259 if (!GenResult)
4260 return GenResult.takeError();
4261
4262 ElemPtr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ElemPtr);
4263 }
4264 TargetElement = Builder.CreateLoad(Ty: RI.ByRefElementType, Ptr: ElemPtr);
4265 }
4266
4267 Builder.CreateStore(Val: TargetElement, Ptr: GlobVal);
4268 break;
4269 }
4270 case EvalKind::Complex: {
4271 Value *SrcRealPtr = Builder.CreateConstInBoundsGEP2_32(
4272 Ty: RI.ElementType, Ptr: ElemPtr, Idx0: 0, Idx1: 0, Name: ".realp");
4273 Value *SrcReal = Builder.CreateLoad(
4274 Ty: RI.ElementType->getStructElementType(N: 0), Ptr: SrcRealPtr, Name: ".real");
4275 Value *SrcImgPtr = Builder.CreateConstInBoundsGEP2_32(
4276 Ty: RI.ElementType, Ptr: ElemPtr, Idx0: 0, Idx1: 1, Name: ".imagp");
4277 Value *SrcImg = Builder.CreateLoad(
4278 Ty: RI.ElementType->getStructElementType(N: 1), Ptr: SrcImgPtr, Name: ".imag");
4279
4280 Value *DestRealPtr = Builder.CreateConstInBoundsGEP2_32(
4281 Ty: RI.ElementType, Ptr: GlobVal, Idx0: 0, Idx1: 0, Name: ".realp");
4282 Value *DestImgPtr = Builder.CreateConstInBoundsGEP2_32(
4283 Ty: RI.ElementType, Ptr: GlobVal, Idx0: 0, Idx1: 1, Name: ".imagp");
4284 Builder.CreateStore(Val: SrcReal, Ptr: DestRealPtr);
4285 Builder.CreateStore(Val: SrcImg, Ptr: DestImgPtr);
4286 break;
4287 }
4288 case EvalKind::Aggregate: {
4289 Value *SizeVal =
4290 Builder.getInt64(C: M.getDataLayout().getTypeStoreSize(Ty: RI.ElementType));
4291 Builder.CreateMemCpy(
4292 Dst: GlobVal, DstAlign: M.getDataLayout().getPrefTypeAlign(Ty: RI.ElementType), Src: ElemPtr,
4293 SrcAlign: M.getDataLayout().getPrefTypeAlign(Ty: RI.ElementType), Size: SizeVal, isVolatile: false);
4294 break;
4295 }
4296 }
4297 }
4298
4299 Builder.CreateRetVoid();
4300 return LtGCFunc;
4301}
4302
4303Expected<Function *> OpenMPIRBuilder::emitListToGlobalReduceFunction(
4304 ArrayRef<ReductionInfo> ReductionInfos, Function *ReduceFn,
4305 Type *ReductionsBufferTy, AttributeList FuncAttrs, ArrayRef<bool> IsByRef) {
4306 IRBuilder<>::InsertPointGuard IPG(Builder);
4307 LLVMContext &Ctx = M.getContext();
4308 FunctionType *FuncTy = FunctionType::get(
4309 Result: Builder.getVoidTy(),
4310 Params: {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},
4311 /* IsVarArg */ isVarArg: false);
4312 Function *LtGRFunc =
4313 Function::Create(Ty: FuncTy, Linkage: GlobalVariable::InternalLinkage,
4314 N: "_omp_reduction_list_to_global_reduce_func", M: &M);
4315 LtGRFunc->setAttributes(FuncAttrs);
4316 LtGRFunc->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
4317 LtGRFunc->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
4318 LtGRFunc->addParamAttr(ArgNo: 2, Kind: Attribute::NoUndef);
4319
4320 BasicBlock *EntryBlock = BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: LtGRFunc);
4321 Builder.SetInsertPoint(EntryBlock);
4322 Builder.SetCurrentDebugLocation(llvm::DebugLoc());
4323
4324 // Buffer: global reduction buffer.
4325 Argument *BufferArg = LtGRFunc->getArg(i: 0);
4326 // Idx: index of the buffer.
4327 Argument *IdxArg = LtGRFunc->getArg(i: 1);
4328 // ReduceList: thread local Reduce list.
4329 Argument *ReduceListArg = LtGRFunc->getArg(i: 2);
4330
4331 Value *BufferArgAlloca = Builder.CreateAlloca(Ty: Builder.getPtrTy(), ArraySize: nullptr,
4332 Name: BufferArg->getName() + ".addr");
4333 Value *IdxArgAlloca = Builder.CreateAlloca(Ty: Builder.getInt32Ty(), ArraySize: nullptr,
4334 Name: IdxArg->getName() + ".addr");
4335 Value *ReduceListArgAlloca = Builder.CreateAlloca(
4336 Ty: Builder.getPtrTy(), ArraySize: nullptr, Name: ReduceListArg->getName() + ".addr");
4337 auto *RedListArrayTy =
4338 ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: ReductionInfos.size());
4339
4340 // 1. Build a list of reduction variables.
4341 // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
4342 Value *LocalReduceList =
4343 Builder.CreateAlloca(Ty: RedListArrayTy, ArraySize: nullptr, Name: ".omp.reduction.red_list");
4344
4345 InsertPointTy AllocaIP(EntryBlock->begin());
4346
4347 Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4348 V: BufferArgAlloca, DestTy: Builder.getPtrTy(),
4349 Name: BufferArgAlloca->getName() + ".ascast");
4350 Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4351 V: IdxArgAlloca, DestTy: Builder.getPtrTy(), Name: IdxArgAlloca->getName() + ".ascast");
4352 Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4353 V: ReduceListArgAlloca, DestTy: Builder.getPtrTy(),
4354 Name: ReduceListArgAlloca->getName() + ".ascast");
4355 Value *LocalReduceListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4356 V: LocalReduceList, DestTy: Builder.getPtrTy(),
4357 Name: LocalReduceList->getName() + ".ascast");
4358
4359 Builder.CreateStore(Val: BufferArg, Ptr: BufferArgAddrCast);
4360 Builder.CreateStore(Val: IdxArg, Ptr: IdxArgAddrCast);
4361 Builder.CreateStore(Val: ReduceListArg, Ptr: ReduceListArgAddrCast);
4362
4363 Value *BufferVal = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: BufferArgAddrCast);
4364 Value *Idxs[] = {Builder.CreateLoad(Ty: Builder.getInt32Ty(), Ptr: IdxArgAddrCast)};
4365 Type *IndexTy = Builder.getIndexTy(
4366 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
4367 for (auto En : enumerate(First&: ReductionInfos)) {
4368 const ReductionInfo &RI = En.value();
4369
4370 Value *TargetElementPtrPtr = Builder.CreateInBoundsGEP(
4371 Ty: RedListArrayTy, Ptr: LocalReduceListAddrCast,
4372 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
4373 Value *BufferVD =
4374 Builder.CreateInBoundsGEP(Ty: ReductionsBufferTy, Ptr: BufferVal, IdxList: Idxs);
4375 // Global = Buffer.VD[Idx];
4376 Value *GlobValPtr = Builder.CreateConstInBoundsGEP2_32(
4377 Ty: ReductionsBufferTy, Ptr: BufferVD, Idx0: 0, Idx1: En.index());
4378
4379 if (!IsByRef.empty() && IsByRef[En.index()] && RI.DataPtrPtrGen) {
4380 // Get source descriptor from the reduce list argument
4381 Value *ReduceList =
4382 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ReduceListArgAddrCast);
4383 Value *SrcElementPtrPtr =
4384 Builder.CreateInBoundsGEP(Ty: RedListArrayTy, Ptr: ReduceList,
4385 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0),
4386 ConstantInt::get(Ty: IndexTy, V: En.index())});
4387 Value *SrcDescriptorAddr =
4388 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: SrcElementPtrPtr);
4389
4390 // Copy descriptor from source and update base_ptr to global buffer data
4391 Expected<Value *> ByRefAlloc = createReductionDescriptorCopy(
4392 AllocaIP, RI, DataPtr: GlobValPtr, SrcDescriptorAddr, DescriptorPtrTy: Builder.getPtrTy());
4393 if (!ByRefAlloc)
4394 return ByRefAlloc.takeError();
4395
4396 Builder.CreateStore(Val: *ByRefAlloc, Ptr: TargetElementPtrPtr);
4397 } else {
4398 Builder.CreateStore(Val: GlobValPtr, Ptr: TargetElementPtrPtr);
4399 }
4400 }
4401
4402 // Call reduce_function(GlobalReduceList, ReduceList)
4403 Value *ReduceList =
4404 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ReduceListArgAddrCast);
4405 createRuntimeFunctionCall(Callee: ReduceFn, Args: {LocalReduceListAddrCast, ReduceList})
4406 ->addFnAttr(Kind: Attribute::NoUnwind);
4407 Builder.CreateRetVoid();
4408 return LtGRFunc;
4409}
4410
4411Expected<Function *> OpenMPIRBuilder::emitGlobalToListCopyFunction(
4412 ArrayRef<ReductionInfo> ReductionInfos, Type *ReductionsBufferTy,
4413 AttributeList FuncAttrs, ArrayRef<bool> IsByRef) {
4414 IRBuilder<>::InsertPointGuard IPG(Builder);
4415 LLVMContext &Ctx = M.getContext();
4416 FunctionType *FuncTy = FunctionType::get(
4417 Result: Builder.getVoidTy(),
4418 Params: {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},
4419 /* IsVarArg */ isVarArg: false);
4420 Function *GtLCFunc =
4421 Function::Create(Ty: FuncTy, Linkage: GlobalVariable::InternalLinkage,
4422 N: "_omp_reduction_global_to_list_copy_func", M: &M);
4423 GtLCFunc->setAttributes(FuncAttrs);
4424 GtLCFunc->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
4425 GtLCFunc->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
4426 GtLCFunc->addParamAttr(ArgNo: 2, Kind: Attribute::NoUndef);
4427
4428 BasicBlock *EntryBlock = BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: GtLCFunc);
4429 Builder.SetInsertPoint(EntryBlock);
4430 Builder.SetCurrentDebugLocation(llvm::DebugLoc());
4431
4432 // Buffer: global reduction buffer.
4433 Argument *BufferArg = GtLCFunc->getArg(i: 0);
4434 // Idx: index of the buffer.
4435 Argument *IdxArg = GtLCFunc->getArg(i: 1);
4436 // ReduceList: thread local Reduce list.
4437 Argument *ReduceListArg = GtLCFunc->getArg(i: 2);
4438
4439 Value *BufferArgAlloca = Builder.CreateAlloca(Ty: Builder.getPtrTy(), ArraySize: nullptr,
4440 Name: BufferArg->getName() + ".addr");
4441 Value *IdxArgAlloca = Builder.CreateAlloca(Ty: Builder.getInt32Ty(), ArraySize: nullptr,
4442 Name: IdxArg->getName() + ".addr");
4443 Value *ReduceListArgAlloca = Builder.CreateAlloca(
4444 Ty: Builder.getPtrTy(), ArraySize: nullptr, Name: ReduceListArg->getName() + ".addr");
4445 Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4446 V: BufferArgAlloca, DestTy: Builder.getPtrTy(),
4447 Name: BufferArgAlloca->getName() + ".ascast");
4448 Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4449 V: IdxArgAlloca, DestTy: Builder.getPtrTy(), Name: IdxArgAlloca->getName() + ".ascast");
4450 Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4451 V: ReduceListArgAlloca, DestTy: Builder.getPtrTy(),
4452 Name: ReduceListArgAlloca->getName() + ".ascast");
4453 Builder.CreateStore(Val: BufferArg, Ptr: BufferArgAddrCast);
4454 Builder.CreateStore(Val: IdxArg, Ptr: IdxArgAddrCast);
4455 Builder.CreateStore(Val: ReduceListArg, Ptr: ReduceListArgAddrCast);
4456
4457 Value *LocalReduceList =
4458 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ReduceListArgAddrCast);
4459 Value *BufferVal = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: BufferArgAddrCast);
4460 Value *Idxs[] = {Builder.CreateLoad(Ty: Builder.getInt32Ty(), Ptr: IdxArgAddrCast)};
4461 Type *IndexTy = Builder.getIndexTy(
4462 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
4463 for (auto En : enumerate(First&: ReductionInfos)) {
4464 const OpenMPIRBuilder::ReductionInfo &RI = En.value();
4465 auto *RedListArrayTy =
4466 ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: ReductionInfos.size());
4467 // Reduce element = LocalReduceList[i]
4468 Value *ElemPtrPtr = Builder.CreateInBoundsGEP(
4469 Ty: RedListArrayTy, Ptr: LocalReduceList,
4470 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
4471 // elemptr = ((CopyType*)(elemptrptr)) + I
4472 Value *ElemPtr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ElemPtrPtr);
4473 // Global = Buffer.VD[Idx];
4474 Value *BufferVD =
4475 Builder.CreateInBoundsGEP(Ty: ReductionsBufferTy, Ptr: BufferVal, IdxList: Idxs);
4476 Value *GlobValPtr = Builder.CreateConstInBoundsGEP2_32(
4477 Ty: ReductionsBufferTy, Ptr: BufferVD, Idx0: 0, Idx1: En.index());
4478
4479 switch (RI.EvaluationKind) {
4480 case EvalKind::Scalar: {
4481 Type *ElemType = RI.ElementType;
4482
4483 if (!IsByRef.empty() && IsByRef[En.index()]) {
4484 ElemType = RI.ByRefElementType;
4485 if (RI.DataPtrPtrGen) {
4486 InsertPointOrErrorTy GenResult =
4487 RI.DataPtrPtrGen(Builder.saveIP(), ElemPtr, ElemPtr);
4488
4489 if (!GenResult)
4490 return GenResult.takeError();
4491
4492 ElemPtr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ElemPtr);
4493 }
4494 }
4495
4496 Value *TargetElement = Builder.CreateLoad(Ty: ElemType, Ptr: GlobValPtr);
4497 Builder.CreateStore(Val: TargetElement, Ptr: ElemPtr);
4498 break;
4499 }
4500 case EvalKind::Complex: {
4501 Value *SrcRealPtr = Builder.CreateConstInBoundsGEP2_32(
4502 Ty: RI.ElementType, Ptr: GlobValPtr, Idx0: 0, Idx1: 0, Name: ".realp");
4503 Value *SrcReal = Builder.CreateLoad(
4504 Ty: RI.ElementType->getStructElementType(N: 0), Ptr: SrcRealPtr, Name: ".real");
4505 Value *SrcImgPtr = Builder.CreateConstInBoundsGEP2_32(
4506 Ty: RI.ElementType, Ptr: GlobValPtr, Idx0: 0, Idx1: 1, Name: ".imagp");
4507 Value *SrcImg = Builder.CreateLoad(
4508 Ty: RI.ElementType->getStructElementType(N: 1), Ptr: SrcImgPtr, Name: ".imag");
4509
4510 Value *DestRealPtr = Builder.CreateConstInBoundsGEP2_32(
4511 Ty: RI.ElementType, Ptr: ElemPtr, Idx0: 0, Idx1: 0, Name: ".realp");
4512 Value *DestImgPtr = Builder.CreateConstInBoundsGEP2_32(
4513 Ty: RI.ElementType, Ptr: ElemPtr, Idx0: 0, Idx1: 1, Name: ".imagp");
4514 Builder.CreateStore(Val: SrcReal, Ptr: DestRealPtr);
4515 Builder.CreateStore(Val: SrcImg, Ptr: DestImgPtr);
4516 break;
4517 }
4518 case EvalKind::Aggregate: {
4519 Value *SizeVal =
4520 Builder.getInt64(C: M.getDataLayout().getTypeStoreSize(Ty: RI.ElementType));
4521 Builder.CreateMemCpy(
4522 Dst: ElemPtr, DstAlign: M.getDataLayout().getPrefTypeAlign(Ty: RI.ElementType),
4523 Src: GlobValPtr, SrcAlign: M.getDataLayout().getPrefTypeAlign(Ty: RI.ElementType),
4524 Size: SizeVal, isVolatile: false);
4525 break;
4526 }
4527 }
4528 }
4529
4530 Builder.CreateRetVoid();
4531 return GtLCFunc;
4532}
4533
4534Expected<Function *> OpenMPIRBuilder::emitGlobalToListReduceFunction(
4535 ArrayRef<ReductionInfo> ReductionInfos, Function *ReduceFn,
4536 Type *ReductionsBufferTy, AttributeList FuncAttrs, ArrayRef<bool> IsByRef) {
4537 IRBuilder<>::InsertPointGuard IPG(Builder);
4538 LLVMContext &Ctx = M.getContext();
4539 auto *FuncTy = FunctionType::get(
4540 Result: Builder.getVoidTy(),
4541 Params: {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},
4542 /* IsVarArg */ isVarArg: false);
4543 Function *GtLRFunc =
4544 Function::Create(Ty: FuncTy, Linkage: GlobalVariable::InternalLinkage,
4545 N: "_omp_reduction_global_to_list_reduce_func", M: &M);
4546 GtLRFunc->setAttributes(FuncAttrs);
4547 GtLRFunc->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
4548 GtLRFunc->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
4549 GtLRFunc->addParamAttr(ArgNo: 2, Kind: Attribute::NoUndef);
4550
4551 BasicBlock *EntryBlock = BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: GtLRFunc);
4552 Builder.SetInsertPoint(EntryBlock);
4553 Builder.SetCurrentDebugLocation(llvm::DebugLoc());
4554
4555 // Buffer: global reduction buffer.
4556 Argument *BufferArg = GtLRFunc->getArg(i: 0);
4557 // Idx: index of the buffer.
4558 Argument *IdxArg = GtLRFunc->getArg(i: 1);
4559 // ReduceList: thread local Reduce list.
4560 Argument *ReduceListArg = GtLRFunc->getArg(i: 2);
4561
4562 Value *BufferArgAlloca = Builder.CreateAlloca(Ty: Builder.getPtrTy(), ArraySize: nullptr,
4563 Name: BufferArg->getName() + ".addr");
4564 Value *IdxArgAlloca = Builder.CreateAlloca(Ty: Builder.getInt32Ty(), ArraySize: nullptr,
4565 Name: IdxArg->getName() + ".addr");
4566 Value *ReduceListArgAlloca = Builder.CreateAlloca(
4567 Ty: Builder.getPtrTy(), ArraySize: nullptr, Name: ReduceListArg->getName() + ".addr");
4568 ArrayType *RedListArrayTy =
4569 ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: ReductionInfos.size());
4570
4571 // 1. Build a list of reduction variables.
4572 // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
4573 Value *LocalReduceList =
4574 Builder.CreateAlloca(Ty: RedListArrayTy, ArraySize: nullptr, Name: ".omp.reduction.red_list");
4575
4576 InsertPointTy AllocaIP(EntryBlock->begin());
4577
4578 Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4579 V: BufferArgAlloca, DestTy: Builder.getPtrTy(),
4580 Name: BufferArgAlloca->getName() + ".ascast");
4581 Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4582 V: IdxArgAlloca, DestTy: Builder.getPtrTy(), Name: IdxArgAlloca->getName() + ".ascast");
4583 Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4584 V: ReduceListArgAlloca, DestTy: Builder.getPtrTy(),
4585 Name: ReduceListArgAlloca->getName() + ".ascast");
4586 Value *ReductionList = Builder.CreatePointerBitCastOrAddrSpaceCast(
4587 V: LocalReduceList, DestTy: Builder.getPtrTy(),
4588 Name: LocalReduceList->getName() + ".ascast");
4589
4590 Builder.CreateStore(Val: BufferArg, Ptr: BufferArgAddrCast);
4591 Builder.CreateStore(Val: IdxArg, Ptr: IdxArgAddrCast);
4592 Builder.CreateStore(Val: ReduceListArg, Ptr: ReduceListArgAddrCast);
4593
4594 Value *BufferVal = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: BufferArgAddrCast);
4595 Value *Idxs[] = {Builder.CreateLoad(Ty: Builder.getInt32Ty(), Ptr: IdxArgAddrCast)};
4596 Type *IndexTy = Builder.getIndexTy(
4597 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
4598 for (auto En : enumerate(First&: ReductionInfos)) {
4599 const ReductionInfo &RI = En.value();
4600
4601 Value *TargetElementPtrPtr = Builder.CreateInBoundsGEP(
4602 Ty: RedListArrayTy, Ptr: ReductionList,
4603 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
4604 // Global = Buffer.VD[Idx];
4605 Value *BufferVD =
4606 Builder.CreateInBoundsGEP(Ty: ReductionsBufferTy, Ptr: BufferVal, IdxList: Idxs);
4607 Value *GlobValPtr = Builder.CreateConstInBoundsGEP2_32(
4608 Ty: ReductionsBufferTy, Ptr: BufferVD, Idx0: 0, Idx1: En.index());
4609
4610 if (!IsByRef.empty() && IsByRef[En.index()] && RI.DataPtrPtrGen) {
4611 // Get source descriptor from the reduce list
4612 Value *ReduceListVal =
4613 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ReduceListArgAddrCast);
4614 Value *SrcElementPtrPtr =
4615 Builder.CreateInBoundsGEP(Ty: RedListArrayTy, Ptr: ReduceListVal,
4616 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0),
4617 ConstantInt::get(Ty: IndexTy, V: En.index())});
4618 Value *SrcDescriptorAddr =
4619 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: SrcElementPtrPtr);
4620
4621 // Copy descriptor from source and update base_ptr to global buffer data
4622 Expected<Value *> ByRefAlloc = createReductionDescriptorCopy(
4623 AllocaIP, RI, DataPtr: GlobValPtr, SrcDescriptorAddr, DescriptorPtrTy: Builder.getPtrTy());
4624 if (!ByRefAlloc)
4625 return ByRefAlloc.takeError();
4626
4627 Builder.CreateStore(Val: *ByRefAlloc, Ptr: TargetElementPtrPtr);
4628 } else {
4629 Builder.CreateStore(Val: GlobValPtr, Ptr: TargetElementPtrPtr);
4630 }
4631 }
4632
4633 // Call reduce_function(ReduceList, GlobalReduceList)
4634 Value *ReduceList =
4635 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ReduceListArgAddrCast);
4636 createRuntimeFunctionCall(Callee: ReduceFn, Args: {ReduceList, ReductionList})
4637 ->addFnAttr(Kind: Attribute::NoUnwind);
4638 Builder.CreateRetVoid();
4639 return GtLRFunc;
4640}
4641
4642std::string OpenMPIRBuilder::getReductionFuncName(StringRef Name) const {
4643 std::string Suffix =
4644 createPlatformSpecificName(Parts: {"omp", "reduction", "reduction_func"});
4645 return (Name + Suffix).str();
4646}
4647
4648Expected<Function *> OpenMPIRBuilder::createReductionFunction(
4649 StringRef ReducerName, ArrayRef<ReductionInfo> ReductionInfos,
4650 ArrayRef<bool> IsByRef, ReductionGenCBKind ReductionGenCBKind,
4651 AttributeList FuncAttrs) {
4652 IRBuilder<>::InsertPointGuard IPG(Builder);
4653 auto *FuncTy = FunctionType::get(Result: Builder.getVoidTy(),
4654 Params: {Builder.getPtrTy(), Builder.getPtrTy()},
4655 /* IsVarArg */ isVarArg: false);
4656 std::string Name = getReductionFuncName(Name: ReducerName);
4657 Function *ReductionFunc =
4658 Function::Create(Ty: FuncTy, Linkage: GlobalVariable::InternalLinkage, N: Name, M: &M);
4659 ReductionFunc->setCallingConv(Config.getRuntimeCC());
4660 ReductionFunc->setAttributes(FuncAttrs);
4661 ReductionFunc->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
4662 ReductionFunc->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
4663 BasicBlock *EntryBB =
4664 BasicBlock::Create(Context&: M.getContext(), Name: "entry", Parent: ReductionFunc);
4665 Builder.SetInsertPoint(EntryBB);
4666 Builder.SetCurrentDebugLocation(llvm::DebugLoc());
4667
4668 // Need to alloca memory here and deal with the pointers before getting
4669 // LHS/RHS pointers out
4670 Value *LHSArrayPtr = nullptr;
4671 Value *RHSArrayPtr = nullptr;
4672 Argument *Arg0 = ReductionFunc->getArg(i: 0);
4673 Argument *Arg1 = ReductionFunc->getArg(i: 1);
4674 Type *Arg0Type = Arg0->getType();
4675 Type *Arg1Type = Arg1->getType();
4676
4677 Value *LHSAlloca =
4678 Builder.CreateAlloca(Ty: Arg0Type, ArraySize: nullptr, Name: Arg0->getName() + ".addr");
4679 Value *RHSAlloca =
4680 Builder.CreateAlloca(Ty: Arg1Type, ArraySize: nullptr, Name: Arg1->getName() + ".addr");
4681 Value *LHSAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4682 V: LHSAlloca, DestTy: Arg0Type, Name: LHSAlloca->getName() + ".ascast");
4683 Value *RHSAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4684 V: RHSAlloca, DestTy: Arg1Type, Name: RHSAlloca->getName() + ".ascast");
4685 Builder.CreateStore(Val: Arg0, Ptr: LHSAddrCast);
4686 Builder.CreateStore(Val: Arg1, Ptr: RHSAddrCast);
4687 LHSArrayPtr = Builder.CreateLoad(Ty: Arg0Type, Ptr: LHSAddrCast);
4688 RHSArrayPtr = Builder.CreateLoad(Ty: Arg1Type, Ptr: RHSAddrCast);
4689
4690 Type *RedArrayTy = ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: ReductionInfos.size());
4691 Type *IndexTy = Builder.getIndexTy(
4692 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
4693 SmallVector<Value *> LHSPtrs, RHSPtrs;
4694 for (auto En : enumerate(First&: ReductionInfos)) {
4695 const ReductionInfo &RI = En.value();
4696 Value *RHSI8PtrPtr = Builder.CreateInBoundsGEP(
4697 Ty: RedArrayTy, Ptr: RHSArrayPtr,
4698 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
4699 Value *RHSI8Ptr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: RHSI8PtrPtr);
4700 Value *RHSPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
4701 V: RHSI8Ptr, DestTy: RI.PrivateVariable->getType(),
4702 Name: RHSI8Ptr->getName() + ".ascast");
4703
4704 Value *LHSI8PtrPtr = Builder.CreateInBoundsGEP(
4705 Ty: RedArrayTy, Ptr: LHSArrayPtr,
4706 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
4707 Value *LHSI8Ptr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: LHSI8PtrPtr);
4708 Value *LHSPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
4709 V: LHSI8Ptr, DestTy: RI.Variable->getType(), Name: LHSI8Ptr->getName() + ".ascast");
4710
4711 if (ReductionGenCBKind == ReductionGenCBKind::Clang) {
4712 LHSPtrs.emplace_back(Args&: LHSPtr);
4713 RHSPtrs.emplace_back(Args&: RHSPtr);
4714 } else {
4715 Value *LHS = LHSPtr;
4716 Value *RHS = RHSPtr;
4717
4718 if (!IsByRef.empty() && !IsByRef[En.index()]) {
4719 LHS = Builder.CreateLoad(Ty: RI.ElementType, Ptr: LHSPtr);
4720 RHS = Builder.CreateLoad(Ty: RI.ElementType, Ptr: RHSPtr);
4721 }
4722
4723 Value *Reduced;
4724 InsertPointOrErrorTy AfterIP =
4725 RI.ReductionGen(Builder.saveIP(), LHS, RHS, Reduced);
4726 if (!AfterIP)
4727 return AfterIP.takeError();
4728 if (!Builder.GetInsertBlock())
4729 return ReductionFunc;
4730
4731 Builder.restoreIP(IP: *AfterIP);
4732
4733 if (!IsByRef.empty() && !IsByRef[En.index()])
4734 Builder.CreateStore(Val: Reduced, Ptr: LHSPtr);
4735 }
4736 }
4737
4738 if (ReductionGenCBKind == ReductionGenCBKind::Clang)
4739 for (auto En : enumerate(First&: ReductionInfos)) {
4740 unsigned Index = En.index();
4741 const ReductionInfo &RI = En.value();
4742 Value *LHSFixupPtr, *RHSFixupPtr;
4743 Builder.restoreIP(IP: RI.ReductionGenClang(
4744 Builder.saveIP(), Index, &LHSFixupPtr, &RHSFixupPtr, ReductionFunc));
4745
4746 // Fix the CallBack code genereated to use the correct Values for the LHS
4747 // and RHS
4748 LHSFixupPtr->replaceUsesWithIf(
4749 New: LHSPtrs[Index], ShouldReplace: [ReductionFunc](const Use &U) {
4750 return cast<Instruction>(Val: U.getUser())->getParent()->getParent() ==
4751 ReductionFunc;
4752 });
4753 RHSFixupPtr->replaceUsesWithIf(
4754 New: RHSPtrs[Index], ShouldReplace: [ReductionFunc](const Use &U) {
4755 return cast<Instruction>(Val: U.getUser())->getParent()->getParent() ==
4756 ReductionFunc;
4757 });
4758 }
4759
4760 Builder.CreateRetVoid();
4761 // Compiling with `-O0`, `alloca`s emitted in non-entry blocks are not hoisted
4762 // to the entry block (this is dones for higher opt levels by later passes in
4763 // the pipeline). This has caused issues because non-entry `alloca`s force the
4764 // function to use dynamic stack allocations and we might run out of scratch
4765 // memory.
4766 hoistNonEntryAllocasToEntryBlock(Func: ReductionFunc);
4767
4768 return ReductionFunc;
4769}
4770
4771static void
4772checkReductionInfos(ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos,
4773 bool IsGPU) {
4774 for (const OpenMPIRBuilder::ReductionInfo &RI : ReductionInfos) {
4775 (void)RI;
4776 assert(RI.Variable && "expected non-null variable");
4777 assert(RI.PrivateVariable && "expected non-null private variable");
4778 assert((RI.ReductionGen || RI.ReductionGenClang) &&
4779 "expected non-null reduction generator callback");
4780 if (!IsGPU) {
4781 assert(
4782 RI.Variable->getType() == RI.PrivateVariable->getType() &&
4783 "expected variables and their private equivalents to have the same "
4784 "type");
4785 }
4786 assert(RI.Variable->getType()->isPointerTy() &&
4787 "expected variables to be pointers");
4788 }
4789}
4790
4791// The atomic cross-team reduction fast path applies when every reduction in the
4792// set can be represented by an atomicrmw. Clang only populates it for scalar
4793// reductions with a supported atomic operator.
4794static bool isAtomicableReductionSet(
4795 ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos) {
4796 return all_of(Range&: ReductionInfos, P: [](const OpenMPIRBuilder::ReductionInfo &RI) {
4797 return static_cast<bool>(RI.AtomicReductionGen);
4798 });
4799}
4800
4801OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createReductionsGPU(
4802 const LocationDescription &Loc, InsertPointTy AllocaIP,
4803 InsertPointTy CodeGenIP, ArrayRef<ReductionInfo> ReductionInfos,
4804 ArrayRef<bool> IsByRef, bool IsNoWait, bool IsTeamsReduction, bool IsSPMD,
4805 ReductionGenCBKind ReductionGenCBKind, std::optional<omp::GV> GridValue,
4806 Value *SrcLocInfo) {
4807 if (!updateToLocation(Loc))
4808 return InsertPointTy();
4809 Builder.restoreIP(IP: CodeGenIP);
4810 Builder.SetCurrentDebugLocation(Loc.DL);
4811 checkReductionInfos(ReductionInfos, /*IsGPU*/ true);
4812 LLVMContext &Ctx = M.getContext();
4813
4814 // Source location for the ident struct
4815 if (!SrcLocInfo) {
4816 uint32_t SrcLocStrSize;
4817 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
4818 SrcLocInfo = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
4819 }
4820
4821 if (ReductionInfos.size() == 0)
4822 return Builder.saveIP();
4823
4824 BasicBlock *ContinuationBlock = nullptr;
4825 if (ReductionGenCBKind != ReductionGenCBKind::Clang) {
4826 // Copied code from createReductions
4827 BasicBlock *InsertBlock = Loc.IP.getNodeParent();
4828 ContinuationBlock = InsertBlock->splitBasicBlock(I: Loc.IP, BBName: "reduce.finalize");
4829 InsertBlock->getTerminator()->eraseFromParent();
4830 Builder.SetInsertPoint(InsertBlock->end());
4831 }
4832
4833 Function *CurFunc = Builder.GetInsertBlock()->getParent();
4834 AttributeList FuncAttrs;
4835 AttrBuilder AttrBldr(Ctx);
4836 for (auto Attr : CurFunc->getAttributes().getFnAttrs())
4837 AttrBldr.addAttribute(A: Attr);
4838 AttrBldr.removeAttribute(Val: Attribute::OptimizeNone);
4839 FuncAttrs = FuncAttrs.addFnAttributes(C&: Ctx, B: AttrBldr);
4840
4841 Expected<Function *> ReductionResult = createReductionFunction(
4842 ReducerName: Builder.GetInsertBlock()->getParent()->getName(), ReductionInfos, IsByRef,
4843 ReductionGenCBKind, FuncAttrs);
4844 if (!ReductionResult)
4845 return ReductionResult.takeError();
4846 Function *ReductionFunc = *ReductionResult;
4847
4848 // Set the grid value in the config needed for lowering later on
4849 if (GridValue.has_value())
4850 Config.setGridValue(GridValue.value());
4851 else
4852 Config.setGridValue(getGridValue(T, Kernel: ReductionFunc));
4853
4854 // Build res = __kmpc_reduce{_nowait}(<gtid>, <n>, sizeof(RedList),
4855 // RedList, shuffle_reduce_func, interwarp_copy_func);
4856 // or
4857 // Build res = __kmpc_reduce_teams_nowait_simple(<loc>, <gtid>, <lck>);
4858 Value *Res;
4859
4860 // 1. Build a list of reduction variables.
4861 // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
4862 auto Size = ReductionInfos.size();
4863 Type *PtrTy = PointerType::get(C&: Ctx, AddressSpace: Config.getDefaultTargetAS());
4864 Type *FuncPtrTy =
4865 Builder.getPtrTy(AddrSpace: M.getDataLayout().getProgramAddressSpace());
4866 Type *RedArrayTy = ArrayType::get(ElementType: PtrTy, NumElements: Size);
4867 Value *ReductionList;
4868 {
4869 IRBuilder<>::InsertPointGuard IPG(Builder);
4870 Builder.restoreIP(IP: AllocaIP);
4871 Value *ReductionListAlloca =
4872 Builder.CreateAlloca(Ty: RedArrayTy, ArraySize: nullptr, Name: ".omp.reduction.red_list");
4873 ReductionList = Builder.CreatePointerBitCastOrAddrSpaceCast(
4874 V: ReductionListAlloca, DestTy: PtrTy, Name: ReductionListAlloca->getName() + ".ascast");
4875 }
4876 Type *IndexTy = Builder.getIndexTy(
4877 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
4878 for (auto En : enumerate(First&: ReductionInfos)) {
4879 const ReductionInfo &RI = En.value();
4880 Value *ElemPtr = Builder.CreateInBoundsGEP(
4881 Ty: RedArrayTy, Ptr: ReductionList,
4882 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
4883
4884 Value *PrivateVar = RI.PrivateVariable;
4885 bool IsByRefElem = !IsByRef.empty() && IsByRef[En.index()];
4886 if (IsByRefElem)
4887 PrivateVar = Builder.CreateLoad(Ty: RI.ElementType, Ptr: PrivateVar);
4888
4889 Value *CastElem =
4890 Builder.CreatePointerBitCastOrAddrSpaceCast(V: PrivateVar, DestTy: PtrTy);
4891 Builder.CreateStore(Val: CastElem, Ptr: ElemPtr);
4892 }
4893 Expected<Function *> SarFunc = emitShuffleAndReduceFunction(
4894 ReductionInfos, ReduceFn: ReductionFunc, FuncAttrs, IsByRef);
4895
4896 if (!SarFunc)
4897 return SarFunc.takeError();
4898
4899 Expected<Function *> CopyResult =
4900 emitInterWarpCopyFunction(Loc, ReductionInfos, FuncAttrs, IsByRef);
4901 if (!CopyResult)
4902 return CopyResult.takeError();
4903 Function *WcFunc = *CopyResult;
4904
4905 Value *RL = Builder.CreatePointerBitCastOrAddrSpaceCast(V: ReductionList, DestTy: PtrTy);
4906
4907 // NOTE: ReductionDataSize is passed as the reduce_data_size argument to
4908 // __kmpc_nvptx_parallel_reduce_nowait_v2, but the runtime implementations do
4909 // not currently use it. It is computed here conservatively as max(element
4910 // sizes) * N rather than the exact sum, which over-calculates the size for
4911 // mixed reduction types but is harmless given the argument is unused.
4912 // TODO: Consider dropping this computation if the runtime API is ever revised
4913 // to remove the unused parameter.
4914 unsigned MaxDataSize = 0;
4915 SmallVector<Type *> ReductionTypeArgs;
4916 for (auto En : enumerate(First&: ReductionInfos)) {
4917 // Use ByRefElementType for by-ref reductions so that MaxDataSize matches
4918 // the actual data size stored in the global reduction buffer, consistent
4919 // with the ReductionsBufferTy struct used for GEP offsets below.
4920 Type *RedTypeArg = (!IsByRef.empty() && IsByRef[En.index()])
4921 ? En.value().ByRefElementType
4922 : En.value().ElementType;
4923 auto Size = M.getDataLayout().getTypeStoreSize(Ty: RedTypeArg);
4924 if (Size > MaxDataSize)
4925 MaxDataSize = Size;
4926 ReductionTypeArgs.emplace_back(Args&: RedTypeArg);
4927 }
4928 Value *ReductionDataSize =
4929 Builder.getInt64(C: MaxDataSize * ReductionInfos.size());
4930
4931 // Helper function to copy thread-local data back to the original reduction
4932 // list.
4933 Function *CopyScratchToListFunc = nullptr;
4934 // Thread-local storage for the reduction variables.
4935 Value *ScratchForCopyBack = nullptr;
4936 // RL pointer to which the final value from the per-thread scratch should be
4937 // copied back. (Basically RL, appropriately casted if necessary.)
4938 Value *RLForCopyBack = RL;
4939
4940 bool IsAtomicReduction =
4941 IsTeamsReduction && isAtomicableReductionSet(ReductionInfos);
4942
4943 if (!IsTeamsReduction) {
4944 Value *SarFuncCast =
4945 Builder.CreatePointerBitCastOrAddrSpaceCast(V: *SarFunc, DestTy: FuncPtrTy);
4946 Value *WcFuncCast =
4947 Builder.CreatePointerBitCastOrAddrSpaceCast(V: WcFunc, DestTy: FuncPtrTy);
4948 Value *Args[] = {SrcLocInfo, ReductionDataSize, RL, SarFuncCast,
4949 WcFuncCast};
4950 Function *Pv2Ptr = getOrCreateRuntimeFunctionPtr(
4951 FnID: RuntimeFunction::OMPRTL___kmpc_nvptx_parallel_reduce_nowait_v2);
4952 Res = createRuntimeFunctionCall(Callee: Pv2Ptr, Args);
4953 } else if (IsAtomicReduction) {
4954 // Atomic cross-team reduction fast path: determine the team's main thread
4955 // that is later to fold its value atomically into the mapped variable.
4956 Function *IsMainThreadFn = getOrCreateRuntimeFunctionPtr(
4957 FnID: RuntimeFunction::OMPRTL___kmpc_is_team_main_thread);
4958 Res = createRuntimeFunctionCall(Callee: IsMainThreadFn, Args: {});
4959 } else {
4960 StructType *ReductionsBufferTy = StructType::create(
4961 Context&: Ctx, Elements: ReductionTypeArgs, Name: "struct._globalized_locals_ty");
4962
4963 Expected<Function *> LtGCFunc = emitListToGlobalCopyFunction(
4964 ReductionInfos, ReductionsBufferTy, FuncAttrs, IsByRef);
4965 if (!LtGCFunc)
4966 return LtGCFunc.takeError();
4967
4968 Expected<Function *> GtLCFunc = emitGlobalToListCopyFunction(
4969 ReductionInfos, ReductionsBufferTy, FuncAttrs, IsByRef);
4970 if (!GtLCFunc)
4971 return GtLCFunc.takeError();
4972
4973 Expected<Function *> GtLRFunc = emitGlobalToListReduceFunction(
4974 ReductionInfos, ReduceFn: ReductionFunc, ReductionsBufferTy, FuncAttrs, IsByRef);
4975 if (!GtLRFunc)
4976 return GtLRFunc.takeError();
4977
4978 // The runtime's cross-team final aggregate uses the storage pointed at by
4979 // its reduce-list argument as per-thread scratch. When the surrounding
4980 // kernel is already in SPMD execution mode, clang emitted each reduction
4981 // private as a per-thread `alloca addrspace(5)`, so the original red_list
4982 // (RL) is already per-thread and nothing else is needed.
4983 //
4984 // When the kernel is in Non-SPMD execution mode at codegen time, clang's
4985 // Generic-mode globalization put the reduction private into team-shared
4986 // LDS. OpenMPOpt may later upgrade the kernel to Generic-SPMD, at which
4987 // point all threads of the last team would race on the shared LDS slot.
4988 // Emit a per-thread scratch buffer and a per-thread RL, copy the team-local
4989 // value in, and hand the per-thread RL to the runtime instead. The writer
4990 // thread copies the final value from that per-thread scratch back to RL
4991 // before running the existing combine path below.
4992
4993 // Thread-local RL (might need localization below before being passed to the
4994 // runtime).
4995 Value *RuntimeRL = RL;
4996
4997 if (!IsSPMD) {
4998 Value *PerThreadScratch;
4999
5000 {
5001 IRBuilder<>::InsertPointGuard IPG(Builder);
5002 Builder.restoreIP(IP: AllocaIP);
5003 // Allocate thread-local buffer for the reduction variables.
5004 Value *PerThreadScratchAlloca =
5005 Builder.CreateAlloca(Ty: ReductionsBufferTy, /*ArraySize=*/nullptr,
5006 Name: ".omp.reduction.scratch");
5007 PerThreadScratch = Builder.CreatePointerBitCastOrAddrSpaceCast(
5008 V: PerThreadScratchAlloca, DestTy: PtrTy,
5009 Name: PerThreadScratchAlloca->getName() + ".ascast");
5010 // Allocate thread-local buffer for the pointers to the reduction
5011 // variables.
5012 Value *PerThreadRedListAlloca =
5013 Builder.CreateAlloca(Ty: RedArrayTy, /*ArraySize=*/nullptr,
5014 Name: ".omp.reduction.per_thread_red_list");
5015 RuntimeRL = Builder.CreatePointerBitCastOrAddrSpaceCast(
5016 V: PerThreadRedListAlloca, DestTy: PtrTy,
5017 Name: PerThreadRedListAlloca->getName() + ".ascast");
5018 }
5019
5020 // Iterate over the reduction variables and copy the team-local value to
5021 // the thread-local buffer.
5022 for (auto En : enumerate(First&: ReductionInfos)) {
5023 const ReductionInfo &RI = En.value();
5024 bool IsByRefElem = !IsByRef.empty() && IsByRef[En.index()];
5025
5026 Value *FieldPtr = Builder.CreateConstInBoundsGEP2_32(
5027 Ty: ReductionsBufferTy, Ptr: PerThreadScratch, Idx0: 0, Idx1: En.index());
5028 Value *Slot = Builder.CreateConstInBoundsGEP2_32(Ty: RedArrayTy, Ptr: RuntimeRL,
5029 Idx0: 0, Idx1: En.index());
5030
5031 Value *RuntimeListEntry = FieldPtr;
5032 if (IsByRefElem && RI.DataPtrPtrGen) {
5033 Value *SrcDescriptor =
5034 Builder.CreateLoad(Ty: RI.ElementType, Ptr: RI.PrivateVariable);
5035 Expected<Value *> Descriptor = createReductionDescriptorCopy(
5036 AllocaIP, RI, DataPtr: FieldPtr, SrcDescriptorAddr: SrcDescriptor, DescriptorPtrTy: PtrTy);
5037 if (!Descriptor)
5038 return Descriptor.takeError();
5039 RuntimeListEntry = *Descriptor;
5040 }
5041 Builder.CreateStore(Val: RuntimeListEntry, Ptr: Slot);
5042 }
5043 // The copy helpers were emitted with default-AS (AS 0) pointer params
5044 // (see emitListToGlobalCopyFunction / emitGlobalToListCopyFunction),
5045 // but PerThreadScratch and RL live in the target's default AS, which
5046 // is non-zero on e.g. SPIRV. (See Config.getDefaultTargetAS().)
5047 Type *CopyArg0Ty = (*LtGCFunc)->getFunctionType()->getParamType(i: 0);
5048 Type *CopyArg2Ty = (*LtGCFunc)->getFunctionType()->getParamType(i: 2);
5049 ScratchForCopyBack = Builder.CreatePointerBitCastOrAddrSpaceCast(
5050 V: PerThreadScratch, DestTy: CopyArg0Ty);
5051 RLForCopyBack =
5052 Builder.CreatePointerBitCastOrAddrSpaceCast(V: RL, DestTy: CopyArg2Ty);
5053 // Use index 0 because there is no array of target values to index into,
5054 // there is only one thread-local memory slot.
5055 // restoreIP above left a stale/empty debug location; this inlinable call
5056 // to a debug-info-bearing helper needs one or the verifier rejects the
5057 // module ("!dbg attachment points at wrong subprogram") after inlining.
5058 Builder.SetCurrentDebugLocation(Loc.DL);
5059 Builder.CreateCall(
5060 Callee: *LtGCFunc, Args: {ScratchForCopyBack, Builder.getInt32(C: 0), RLForCopyBack});
5061 CopyScratchToListFunc = *GtLCFunc;
5062 }
5063
5064 Value *Args3[] = {SrcLocInfo, RuntimeRL, *SarFunc, WcFunc,
5065 *LtGCFunc, *GtLCFunc, *GtLRFunc};
5066
5067 Function *TeamsReduceFn = getOrCreateRuntimeFunctionPtr(
5068 FnID: RuntimeFunction::OMPRTL___kmpc_gpu_xteam_reduce_nowait);
5069 Res = createRuntimeFunctionCall(Callee: TeamsReduceFn, Args: Args3);
5070 }
5071
5072 // 5. Build if (res == 1)
5073 BasicBlock *ExitBB = BasicBlock::Create(Context&: Ctx, Name: ".omp.reduction.done");
5074 BasicBlock *ThenBB = BasicBlock::Create(Context&: Ctx, Name: ".omp.reduction.then");
5075 Value *Cond = Builder.CreateICmpEQ(LHS: Res, RHS: Builder.getInt32(C: 1));
5076 Builder.CreateCondBr(Cond, True: ThenBB, False: ExitBB);
5077
5078 // 6. Build then branch: where we have reduced values in the master
5079 // thread in each team.
5080 // __kmpc_end_reduce{_nowait}(<gtid>);
5081 // break;
5082 emitBlock(BB: ThenBB, CurFn: CurFunc);
5083
5084 // Copy the writer thread's per-thread scratch result back into the original
5085 // red-list storage before the existing combine path reads RI.PrivateVariable.
5086 // Set a debug location: this inlinable call to a debug-info-bearing helper
5087 // needs one or the verifier rejects the module after inlining.
5088 if (ScratchForCopyBack) {
5089 Builder.SetCurrentDebugLocation(Loc.DL);
5090 Builder.CreateCall(
5091 Callee: CopyScratchToListFunc,
5092 Args: {ScratchForCopyBack, Builder.getInt32(C: 0), RLForCopyBack});
5093 }
5094
5095 // Add emission of __kmpc_end_reduce{_nowait}(<gtid>);
5096 for (auto En : enumerate(First&: ReductionInfos)) {
5097 const ReductionInfo &RI = En.value();
5098
5099 // Atomic cross-team fast path: each team's main thread folds its
5100 // team-reduced value directly into the mapped reduction variable with a
5101 // single atomicrmw.
5102 if (IsAtomicReduction) {
5103 InsertPointOrErrorTy AfterIP = RI.AtomicReductionGen(
5104 Builder.saveIP(), RI.ElementType, RI.Variable, RI.PrivateVariable);
5105 if (!AfterIP)
5106 return AfterIP.takeError();
5107 Builder.restoreIP(IP: *AfterIP);
5108 continue;
5109 }
5110
5111 Type *ValueType = RI.ElementType;
5112 Value *RedValue = RI.Variable;
5113
5114 Value *RHS =
5115 Builder.CreatePointerBitCastOrAddrSpaceCast(V: RI.PrivateVariable, DestTy: PtrTy);
5116
5117 if (ReductionGenCBKind == ReductionGenCBKind::Clang) {
5118 Value *LHSPtr, *RHSPtr;
5119 Builder.restoreIP(IP: RI.ReductionGenClang(Builder.saveIP(), En.index(),
5120 &LHSPtr, &RHSPtr, CurFunc));
5121
5122 // Fix the CallBack code genereated to use the correct Values for the LHS
5123 // and RHS. Cast to match types before replacing (necessary to handle
5124 // different address spaces).
5125 if (LHSPtr->getType() != RedValue->getType())
5126 RedValue = Builder.CreatePointerBitCastOrAddrSpaceCast(
5127 V: RedValue, DestTy: LHSPtr->getType());
5128 if (RHSPtr->getType() != RHS->getType())
5129 RHS =
5130 Builder.CreatePointerBitCastOrAddrSpaceCast(V: RHS, DestTy: RHSPtr->getType());
5131
5132 LHSPtr->replaceUsesWithIf(New: RedValue, ShouldReplace: [ReductionFunc](const Use &U) {
5133 return cast<Instruction>(Val: U.getUser())->getParent()->getParent() ==
5134 ReductionFunc;
5135 });
5136 RHSPtr->replaceUsesWithIf(New: RHS, ShouldReplace: [ReductionFunc](const Use &U) {
5137 return cast<Instruction>(Val: U.getUser())->getParent()->getParent() ==
5138 ReductionFunc;
5139 });
5140 } else {
5141 if (IsByRef.empty() || !IsByRef[En.index()]) {
5142 RedValue = Builder.CreateLoad(Ty: ValueType, Ptr: RI.Variable,
5143 Name: "red.value." + Twine(En.index()));
5144 }
5145 Value *PrivateRedValue = Builder.CreateLoad(
5146 Ty: ValueType, Ptr: RHS, Name: "red.private.value" + Twine(En.index()));
5147 Value *Reduced;
5148 InsertPointOrErrorTy AfterIP =
5149 RI.ReductionGen(Builder.saveIP(), RedValue, PrivateRedValue, Reduced);
5150 if (!AfterIP)
5151 return AfterIP.takeError();
5152 Builder.restoreIP(IP: *AfterIP);
5153
5154 if (!IsByRef.empty() && !IsByRef[En.index()])
5155 Builder.CreateStore(Val: Reduced, Ptr: RI.Variable);
5156 }
5157 }
5158 emitBlock(BB: ExitBB, CurFn: CurFunc);
5159 if (ContinuationBlock) {
5160 Builder.CreateBr(Dest: ContinuationBlock);
5161 Builder.SetInsertPoint(ContinuationBlock);
5162 }
5163 Config.setEmitLLVMUsed();
5164
5165 return Builder.saveIP();
5166}
5167
5168static Function *getFreshReductionFunc(Module &M) {
5169 Type *VoidTy = Type::getVoidTy(C&: M.getContext());
5170 Type *Int8PtrTy = PointerType::getUnqual(C&: M.getContext());
5171 auto *FuncTy =
5172 FunctionType::get(Result: VoidTy, Params: {Int8PtrTy, Int8PtrTy}, /* IsVarArg */ isVarArg: false);
5173 return Function::Create(Ty: FuncTy, Linkage: GlobalVariable::InternalLinkage,
5174 N: ".omp.reduction.func", M: &M);
5175}
5176
5177static Error populateReductionFunction(
5178 Function *ReductionFunc,
5179 ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos,
5180 IRBuilder<> &Builder, ArrayRef<bool> IsByRef, bool IsGPU) {
5181 IRBuilder<>::InsertPointGuard IPG(Builder);
5182 Module *Module = ReductionFunc->getParent();
5183 BasicBlock *ReductionFuncBlock =
5184 BasicBlock::Create(Context&: Module->getContext(), Name: "", Parent: ReductionFunc);
5185 Builder.SetInsertPoint(ReductionFuncBlock);
5186 Builder.SetCurrentDebugLocation(llvm::DebugLoc());
5187 Value *LHSArrayPtr = nullptr;
5188 Value *RHSArrayPtr = nullptr;
5189 if (IsGPU) {
5190 // Need to alloca memory here and deal with the pointers before getting
5191 // LHS/RHS pointers out
5192 //
5193 Argument *Arg0 = ReductionFunc->getArg(i: 0);
5194 Argument *Arg1 = ReductionFunc->getArg(i: 1);
5195 Type *Arg0Type = Arg0->getType();
5196 Type *Arg1Type = Arg1->getType();
5197
5198 Value *LHSAlloca =
5199 Builder.CreateAlloca(Ty: Arg0Type, ArraySize: nullptr, Name: Arg0->getName() + ".addr");
5200 Value *RHSAlloca =
5201 Builder.CreateAlloca(Ty: Arg1Type, ArraySize: nullptr, Name: Arg1->getName() + ".addr");
5202 Value *LHSAddrCast =
5203 Builder.CreatePointerBitCastOrAddrSpaceCast(V: LHSAlloca, DestTy: Arg0Type);
5204 Value *RHSAddrCast =
5205 Builder.CreatePointerBitCastOrAddrSpaceCast(V: RHSAlloca, DestTy: Arg1Type);
5206 Builder.CreateStore(Val: Arg0, Ptr: LHSAddrCast);
5207 Builder.CreateStore(Val: Arg1, Ptr: RHSAddrCast);
5208 LHSArrayPtr = Builder.CreateLoad(Ty: Arg0Type, Ptr: LHSAddrCast);
5209 RHSArrayPtr = Builder.CreateLoad(Ty: Arg1Type, Ptr: RHSAddrCast);
5210 } else {
5211 LHSArrayPtr = ReductionFunc->getArg(i: 0);
5212 RHSArrayPtr = ReductionFunc->getArg(i: 1);
5213 }
5214
5215 unsigned NumReductions = ReductionInfos.size();
5216 Type *RedArrayTy = ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: NumReductions);
5217
5218 for (auto En : enumerate(First&: ReductionInfos)) {
5219 const OpenMPIRBuilder::ReductionInfo &RI = En.value();
5220 Value *LHSI8PtrPtr = Builder.CreateConstInBoundsGEP2_64(
5221 Ty: RedArrayTy, Ptr: LHSArrayPtr, Idx0: 0, Idx1: En.index());
5222 Value *LHSI8Ptr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: LHSI8PtrPtr);
5223 Value *LHSPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
5224 V: LHSI8Ptr, DestTy: RI.Variable->getType());
5225 Value *LHS = Builder.CreateLoad(Ty: RI.ElementType, Ptr: LHSPtr);
5226 Value *RHSI8PtrPtr = Builder.CreateConstInBoundsGEP2_64(
5227 Ty: RedArrayTy, Ptr: RHSArrayPtr, Idx0: 0, Idx1: En.index());
5228 Value *RHSI8Ptr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: RHSI8PtrPtr);
5229 Value *RHSPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
5230 V: RHSI8Ptr, DestTy: RI.PrivateVariable->getType());
5231 Value *RHS = Builder.CreateLoad(Ty: RI.ElementType, Ptr: RHSPtr);
5232 Value *Reduced;
5233 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
5234 RI.ReductionGen(Builder.saveIP(), LHS, RHS, Reduced);
5235 if (!AfterIP)
5236 return AfterIP.takeError();
5237
5238 Builder.restoreIP(IP: *AfterIP);
5239 // TODO: Consider flagging an error.
5240 if (!Builder.GetInsertBlock())
5241 return Error::success();
5242
5243 // store is inside of the reduction region when using by-ref
5244 if (!IsByRef[En.index()])
5245 Builder.CreateStore(Val: Reduced, Ptr: LHSPtr);
5246 }
5247 Builder.CreateRetVoid();
5248 return Error::success();
5249}
5250
5251OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createReductions(
5252 const LocationDescription &Loc, InsertPointTy AllocaIP,
5253 ArrayRef<ReductionInfo> ReductionInfos, ArrayRef<bool> IsByRef,
5254 bool IsNoWait, bool IsTeamsReduction) {
5255 assert(ReductionInfos.size() == IsByRef.size());
5256 if (Config.isGPU())
5257 return createReductionsGPU(Loc, AllocaIP, CodeGenIP: Builder.saveIP(), ReductionInfos,
5258 IsByRef, IsNoWait, IsTeamsReduction);
5259
5260 checkReductionInfos(ReductionInfos, /*IsGPU*/ false);
5261
5262 if (!updateToLocation(Loc))
5263 return InsertPointTy();
5264
5265 if (ReductionInfos.size() == 0)
5266 return Builder.saveIP();
5267
5268 BasicBlock *InsertBlock = Loc.IP.getNodeParent();
5269 BasicBlock *ContinuationBlock =
5270 InsertBlock->splitBasicBlock(I: Loc.IP, BBName: "reduce.finalize");
5271 InsertBlock->getTerminator()->eraseFromParent();
5272
5273 // Create and populate array of type-erased pointers to private reduction
5274 // values.
5275 unsigned NumReductions = ReductionInfos.size();
5276 Type *RedArrayTy = ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: NumReductions);
5277 Builder.SetInsertPoint(AllocaIP.getNodeParent()->getTerminator());
5278 Value *RedArray = Builder.CreateAlloca(Ty: RedArrayTy, ArraySize: nullptr, Name: "red.array");
5279
5280 Builder.SetInsertPoint(InsertBlock->end());
5281 // Emitting the alloca moved the insertion point into the alloca block and
5282 // can clear the debug loc. Restore back to Loc.DL.
5283 Builder.SetCurrentDebugLocation(Loc.DL);
5284
5285 for (auto En : enumerate(First&: ReductionInfos)) {
5286 unsigned Index = En.index();
5287 const ReductionInfo &RI = En.value();
5288 Value *RedArrayElemPtr = Builder.CreateConstInBoundsGEP2_64(
5289 Ty: RedArrayTy, Ptr: RedArray, Idx0: 0, Idx1: Index, Name: "red.array.elem." + Twine(Index));
5290 Builder.CreateStore(Val: RI.PrivateVariable, Ptr: RedArrayElemPtr);
5291 }
5292
5293 // Emit a call to the runtime function that orchestrates the reduction.
5294 // Declare the reduction function in the process.
5295 Type *IndexTy = Builder.getIndexTy(
5296 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
5297 Function *Func = Builder.GetInsertBlock()->getParent();
5298 Module *Module = Func->getParent();
5299 uint32_t SrcLocStrSize;
5300 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
5301 bool CanGenerateAtomic = all_of(Range&: ReductionInfos, P: [](const ReductionInfo &RI) {
5302 return RI.AtomicReductionGen;
5303 });
5304 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize,
5305 LocFlags: CanGenerateAtomic
5306 ? IdentFlag::OMP_IDENT_FLAG_ATOMIC_REDUCE
5307 : IdentFlag(0));
5308 Value *ThreadId = getOrCreateThreadID(Ident);
5309 Constant *NumVariables = Builder.getInt32(C: NumReductions);
5310 const DataLayout &DL = Module->getDataLayout();
5311 unsigned RedArrayByteSize = DL.getTypeStoreSize(Ty: RedArrayTy);
5312 Constant *RedArraySize = ConstantInt::get(Ty: IndexTy, V: RedArrayByteSize);
5313 Function *ReductionFunc = getFreshReductionFunc(M&: *Module);
5314 Value *Lock = getOMPCriticalRegionLock(CriticalName: ".reduction");
5315 Function *ReduceFunc = getOrCreateRuntimeFunctionPtr(
5316 FnID: IsNoWait ? RuntimeFunction::OMPRTL___kmpc_reduce_nowait
5317 : RuntimeFunction::OMPRTL___kmpc_reduce);
5318 CallInst *ReduceCall =
5319 createRuntimeFunctionCall(Callee: ReduceFunc,
5320 Args: {Ident, ThreadId, NumVariables, RedArraySize,
5321 RedArray, ReductionFunc, Lock},
5322 Name: "reduce");
5323
5324 // Create final reduction entry blocks for the atomic and non-atomic case.
5325 // Emit IR that dispatches control flow to one of the blocks based on the
5326 // reduction supporting the atomic mode.
5327 BasicBlock *NonAtomicRedBlock =
5328 BasicBlock::Create(Context&: Module->getContext(), Name: "reduce.switch.nonatomic", Parent: Func);
5329 BasicBlock *AtomicRedBlock =
5330 BasicBlock::Create(Context&: Module->getContext(), Name: "reduce.switch.atomic", Parent: Func);
5331 SwitchInst *Switch =
5332 Builder.CreateSwitch(V: ReduceCall, Dest: ContinuationBlock, /* NumCases */ 2);
5333 Switch->addCase(OnVal: Builder.getInt32(C: 1), Dest: NonAtomicRedBlock);
5334 Switch->addCase(OnVal: Builder.getInt32(C: 2), Dest: AtomicRedBlock);
5335
5336 // Populate the non-atomic reduction using the elementwise reduction function.
5337 // This loads the elements from the global and private variables and reduces
5338 // them before storing back the result to the global variable.
5339 Builder.SetInsertPoint(NonAtomicRedBlock);
5340 for (auto En : enumerate(First&: ReductionInfos)) {
5341 const ReductionInfo &RI = En.value();
5342 Type *ValueType = RI.ElementType;
5343 // We have one less load for by-ref case because that load is now inside of
5344 // the reduction region
5345 Value *RedValue = RI.Variable;
5346 if (!IsByRef[En.index()]) {
5347 RedValue = Builder.CreateLoad(Ty: ValueType, Ptr: RI.Variable,
5348 Name: "red.value." + Twine(En.index()));
5349 }
5350 Value *PrivateRedValue =
5351 Builder.CreateLoad(Ty: ValueType, Ptr: RI.PrivateVariable,
5352 Name: "red.private.value." + Twine(En.index()));
5353 Value *Reduced;
5354 InsertPointOrErrorTy AfterIP =
5355 RI.ReductionGen(Builder.saveIP(), RedValue, PrivateRedValue, Reduced);
5356 if (!AfterIP)
5357 return AfterIP.takeError();
5358 Builder.restoreIP(IP: *AfterIP);
5359
5360 if (!Builder.GetInsertBlock())
5361 return InsertPointTy();
5362 // for by-ref case, the load is inside of the reduction region
5363 if (!IsByRef[En.index()])
5364 Builder.CreateStore(Val: Reduced, Ptr: RI.Variable);
5365 }
5366 Function *EndReduceFunc = getOrCreateRuntimeFunctionPtr(
5367 FnID: IsNoWait ? RuntimeFunction::OMPRTL___kmpc_end_reduce_nowait
5368 : RuntimeFunction::OMPRTL___kmpc_end_reduce);
5369 createRuntimeFunctionCall(Callee: EndReduceFunc, Args: {Ident, ThreadId, Lock});
5370 Builder.CreateBr(Dest: ContinuationBlock);
5371
5372 // Populate the atomic reduction using the atomic elementwise reduction
5373 // function. There are no loads/stores here because they will be happening
5374 // inside the atomic elementwise reduction.
5375 Builder.SetInsertPoint(AtomicRedBlock);
5376 if (CanGenerateAtomic && llvm::none_of(Range&: IsByRef, P: [](bool P) { return P; })) {
5377 for (const ReductionInfo &RI : ReductionInfos) {
5378 InsertPointOrErrorTy AfterIP = RI.AtomicReductionGen(
5379 Builder.saveIP(), RI.ElementType, RI.Variable, RI.PrivateVariable);
5380 if (!AfterIP)
5381 return AfterIP.takeError();
5382 Builder.restoreIP(IP: *AfterIP);
5383 if (!Builder.GetInsertBlock())
5384 return InsertPointTy();
5385 }
5386 Builder.CreateBr(Dest: ContinuationBlock);
5387 } else {
5388 Builder.CreateUnreachable();
5389 }
5390
5391 // Populate the outlined reduction function using the elementwise reduction
5392 // function. Partial values are extracted from the type-erased array of
5393 // pointers to private variables.
5394 Error Err = populateReductionFunction(ReductionFunc, ReductionInfos, Builder,
5395 IsByRef, /*isGPU=*/IsGPU: false);
5396 if (Err)
5397 return Err;
5398
5399 if (!Builder.GetInsertBlock())
5400 return InsertPointTy();
5401
5402 Builder.SetInsertPoint(ContinuationBlock);
5403 return Builder.saveIP();
5404}
5405
5406OpenMPIRBuilder::InsertPointOrErrorTy
5407OpenMPIRBuilder::createMaster(const LocationDescription &Loc,
5408 BodyGenCallbackTy BodyGenCB,
5409 FinalizeCallbackTy FiniCB) {
5410 if (!updateToLocation(Loc))
5411 return Loc.IP;
5412
5413 Directive OMPD = Directive::OMPD_master;
5414 uint32_t SrcLocStrSize;
5415 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
5416 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
5417 Value *ThreadId = getOrCreateThreadID(Ident);
5418 Value *Args[] = {Ident, ThreadId};
5419
5420 Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_master);
5421 Instruction *EntryCall = createRuntimeFunctionCall(Callee: EntryRTLFn, Args);
5422
5423 Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_end_master);
5424 Instruction *ExitCall = createRuntimeFunctionCall(Callee: ExitRTLFn, Args);
5425
5426 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
5427 /*Conditional*/ true, /*hasFinalize*/ HasFinalize: true);
5428}
5429
5430OpenMPIRBuilder::InsertPointOrErrorTy
5431OpenMPIRBuilder::createMasked(const LocationDescription &Loc,
5432 BodyGenCallbackTy BodyGenCB,
5433 FinalizeCallbackTy FiniCB, Value *Filter) {
5434 IRBuilder<>::InsertPointGuard IPG(Builder);
5435 if (!updateToLocation(Loc))
5436 return Loc.IP;
5437
5438 Directive OMPD = Directive::OMPD_masked;
5439 uint32_t SrcLocStrSize;
5440 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
5441 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
5442 Value *ThreadId = getOrCreateThreadID(Ident);
5443 Value *Args[] = {Ident, ThreadId, Filter};
5444 Value *ArgsEnd[] = {Ident, ThreadId};
5445
5446 Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_masked);
5447 Instruction *EntryCall = createRuntimeFunctionCall(Callee: EntryRTLFn, Args);
5448
5449 Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_end_masked);
5450 Instruction *ExitCall = createRuntimeFunctionCall(Callee: ExitRTLFn, Args: ArgsEnd);
5451
5452 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
5453 /*Conditional*/ true, /*hasFinalize*/ HasFinalize: true);
5454}
5455
5456static llvm::CallInst *emitNoUnwindRuntimeCall(IRBuilder<> &Builder,
5457 llvm::FunctionCallee Callee,
5458 ArrayRef<llvm::Value *> Args,
5459 const llvm::Twine &Name) {
5460 llvm::CallInst *Call = Builder.CreateCall(
5461 Callee, Args, OpBundles: SmallVector<llvm::OperandBundleDef, 1>(), Name);
5462 Call->setDoesNotThrow();
5463 return Call;
5464}
5465
5466// Expects input basic block is dominated by BeforeScanBB.
5467// Once Scan directive is encountered, the code after scan directive should be
5468// dominated by AfterScanBB. Scan directive splits the code sequence to
5469// scan and input phase. Based on whether inclusive or exclusive
5470// clause is used in the scan directive and whether input loop or scan loop
5471// is lowered, it adds jumps to input and scan phase. First Scan loop is the
5472// input loop and second is the scan loop. The code generated handles only
5473// inclusive scans now.
5474OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createScan(
5475 const LocationDescription &Loc, InsertPointTy AllocaIP,
5476 ArrayRef<llvm::Value *> ScanVars, ArrayRef<llvm::Type *> ScanVarsType,
5477 bool IsInclusive, ScanInfo *ScanRedInfo) {
5478 if (ScanRedInfo->OMPFirstScanLoop) {
5479 llvm::Error Err = emitScanBasedDirectiveDeclsIR(AllocaIP, ScanVars,
5480 ScanVarsType, ScanRedInfo);
5481 if (Err)
5482 return Err;
5483 }
5484 if (!updateToLocation(Loc))
5485 return Loc.IP;
5486
5487 llvm::Value *IV = ScanRedInfo->IV;
5488
5489 if (ScanRedInfo->OMPFirstScanLoop) {
5490 // Emit buffer[i] = red; at the end of the input phase.
5491 for (size_t i = 0; i < ScanVars.size(); i++) {
5492 Value *BuffPtr = (*(ScanRedInfo->ScanBuffPtrs))[ScanVars[i]];
5493 Value *Buff = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: BuffPtr);
5494 Type *DestTy = ScanVarsType[i];
5495 Value *Val = Builder.CreateInBoundsGEP(Ty: DestTy, Ptr: Buff, IdxList: IV, Name: "arrayOffset");
5496 Value *Src = Builder.CreateLoad(Ty: DestTy, Ptr: ScanVars[i]);
5497
5498 Builder.CreateStore(Val: Src, Ptr: Val);
5499 }
5500 }
5501 Builder.CreateBr(Dest: ScanRedInfo->OMPScanLoopExit);
5502 emitBlock(BB: ScanRedInfo->OMPScanDispatch,
5503 CurFn: Builder.GetInsertBlock()->getParent());
5504
5505 if (!ScanRedInfo->OMPFirstScanLoop) {
5506 IV = ScanRedInfo->IV;
5507 // Emit red = buffer[i]; at the entrance to the scan phase.
5508 // TODO: if exclusive scan, the red = buffer[i-1] needs to be updated.
5509 for (size_t i = 0; i < ScanVars.size(); i++) {
5510 Value *BuffPtr = (*(ScanRedInfo->ScanBuffPtrs))[ScanVars[i]];
5511 Value *Buff = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: BuffPtr);
5512 Type *DestTy = ScanVarsType[i];
5513 Value *SrcPtr =
5514 Builder.CreateInBoundsGEP(Ty: DestTy, Ptr: Buff, IdxList: IV, Name: "arrayOffset");
5515 Value *Src = Builder.CreateLoad(Ty: DestTy, Ptr: SrcPtr);
5516 Builder.CreateStore(Val: Src, Ptr: ScanVars[i]);
5517 }
5518 }
5519
5520 // TODO: Update it to CreateBr and remove dead blocks
5521 llvm::Value *CmpI = Builder.getInt1(V: true);
5522 if (ScanRedInfo->OMPFirstScanLoop == IsInclusive) {
5523 Builder.CreateCondBr(Cond: CmpI, True: ScanRedInfo->OMPBeforeScanBlock,
5524 False: ScanRedInfo->OMPAfterScanBlock);
5525 } else {
5526 Builder.CreateCondBr(Cond: CmpI, True: ScanRedInfo->OMPAfterScanBlock,
5527 False: ScanRedInfo->OMPBeforeScanBlock);
5528 }
5529 emitBlock(BB: ScanRedInfo->OMPAfterScanBlock,
5530 CurFn: Builder.GetInsertBlock()->getParent());
5531 Builder.SetInsertPoint(ScanRedInfo->OMPAfterScanBlock);
5532 return Builder.saveIP();
5533}
5534
5535Error OpenMPIRBuilder::emitScanBasedDirectiveDeclsIR(
5536 InsertPointTy AllocaIP, ArrayRef<Value *> ScanVars,
5537 ArrayRef<Type *> ScanVarsType, ScanInfo *ScanRedInfo) {
5538
5539 Builder.restoreIP(IP: AllocaIP);
5540 // Create the shared pointer at alloca IP.
5541 for (size_t i = 0; i < ScanVars.size(); i++) {
5542 llvm::Value *BuffPtr =
5543 Builder.CreateAlloca(Ty: Builder.getPtrTy(), ArraySize: nullptr, Name: "vla");
5544 (*(ScanRedInfo->ScanBuffPtrs))[ScanVars[i]] = BuffPtr;
5545 }
5546
5547 // Allocate temporary buffer by master thread
5548 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
5549 ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
5550 Builder.restoreIP(IP: CodeGenIP);
5551 Value *AllocSpan =
5552 Builder.CreateAdd(LHS: ScanRedInfo->Span, RHS: Builder.getInt32(C: 1));
5553 for (size_t i = 0; i < ScanVars.size(); i++) {
5554 Type *IntPtrTy = Builder.getInt32Ty();
5555 Value *Allocsize = Builder.CreateTypeSize(
5556 Ty: IntPtrTy, Size: M.getDataLayout().getTypeAllocSize(Ty: ScanVarsType[i]));
5557 Value *Buff =
5558 Builder.CreateMalloc(IntPtrTy, AllocSize: Allocsize, ArraySize: AllocSpan, MallocF: nullptr, Name: "arr");
5559 Builder.CreateStore(Val: Buff, Ptr: (*(ScanRedInfo->ScanBuffPtrs))[ScanVars[i]]);
5560 }
5561 return Error::success();
5562 };
5563 // TODO: Perform finalization actions for variables. This has to be
5564 // called for variables which have destructors/finalizers.
5565 auto FiniCB = [&](InsertPointTy CodeGenIP) { return llvm::Error::success(); };
5566
5567 Builder.SetInsertPoint(ScanRedInfo->OMPScanInit->getTerminator());
5568 llvm::Value *FilterVal = Builder.getInt32(C: 0);
5569 llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
5570 createMasked(Loc: Builder, BodyGenCB, FiniCB, Filter: FilterVal);
5571
5572 if (!AfterIP)
5573 return AfterIP.takeError();
5574 Builder.restoreIP(IP: *AfterIP);
5575 BasicBlock *InputBB = Builder.GetInsertBlock();
5576 if (InputBB->hasTerminator())
5577 Builder.SetInsertPoint(InputBB->getTerminator());
5578 AfterIP = createBarrier(Loc: Builder, Kind: llvm::omp::OMPD_barrier);
5579 if (!AfterIP)
5580 return AfterIP.takeError();
5581 Builder.restoreIP(IP: *AfterIP);
5582
5583 return Error::success();
5584}
5585
5586Error OpenMPIRBuilder::emitScanBasedDirectiveFinalsIR(
5587 ArrayRef<ReductionInfo> ReductionInfos, ScanInfo *ScanRedInfo) {
5588 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
5589 ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
5590 Builder.restoreIP(IP: CodeGenIP);
5591 for (ReductionInfo RedInfo : ReductionInfos) {
5592 Value *PrivateVar = RedInfo.PrivateVariable;
5593 Value *OrigVar = RedInfo.Variable;
5594 Value *BuffPtr = (*(ScanRedInfo->ScanBuffPtrs))[PrivateVar];
5595 Value *Buff = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: BuffPtr);
5596
5597 Type *SrcTy = RedInfo.ElementType;
5598 Value *Val = Builder.CreateInBoundsGEP(Ty: SrcTy, Ptr: Buff, IdxList: ScanRedInfo->Span,
5599 Name: "arrayOffset");
5600 Value *Src = Builder.CreateLoad(Ty: SrcTy, Ptr: Val);
5601
5602 Builder.CreateStore(Val: Src, Ptr: OrigVar);
5603 Builder.CreateFree(Source: Buff);
5604 }
5605 return Error::success();
5606 };
5607 // TODO: Perform finalization actions for variables. This has to be
5608 // called for variables which have destructors/finalizers.
5609 auto FiniCB = [&](InsertPointTy CodeGenIP) { return llvm::Error::success(); };
5610
5611 if (Instruction *TI = ScanRedInfo->OMPScanFinish->getTerminatorOrNull())
5612 Builder.SetInsertPoint(TI);
5613 else
5614 Builder.SetInsertPoint(ScanRedInfo->OMPScanFinish);
5615
5616 llvm::Value *FilterVal = Builder.getInt32(C: 0);
5617 llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
5618 createMasked(Loc: Builder, BodyGenCB, FiniCB, Filter: FilterVal);
5619
5620 if (!AfterIP)
5621 return AfterIP.takeError();
5622 Builder.restoreIP(IP: *AfterIP);
5623 BasicBlock *InputBB = Builder.GetInsertBlock();
5624 if (InputBB->hasTerminator())
5625 Builder.SetInsertPoint(InputBB->getTerminator());
5626 AfterIP = createBarrier(Loc: Builder, Kind: llvm::omp::OMPD_barrier);
5627 if (!AfterIP)
5628 return AfterIP.takeError();
5629 Builder.restoreIP(IP: *AfterIP);
5630 return Error::success();
5631}
5632
5633OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
5634 const LocationDescription &Loc,
5635 ArrayRef<llvm::OpenMPIRBuilder::ReductionInfo> ReductionInfos,
5636 ScanInfo *ScanRedInfo) {
5637
5638 if (!updateToLocation(Loc))
5639 return Loc.IP;
5640 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
5641 ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
5642 Builder.restoreIP(IP: CodeGenIP);
5643 Function *CurFn = Builder.GetInsertBlock()->getParent();
5644 // for (int k = 0; k <= ceil(log2(n)); ++k)
5645 llvm::BasicBlock *LoopBB =
5646 BasicBlock::Create(Context&: CurFn->getContext(), Name: "omp.outer.log.scan.body");
5647 llvm::BasicBlock *ExitBB =
5648 splitBB(Builder, CreateBranch: false, Name: "omp.outer.log.scan.exit");
5649 llvm::Function *F = llvm::Intrinsic::getOrInsertDeclaration(
5650 M: Builder.getModule(), id: (llvm::Intrinsic::ID)llvm::Intrinsic::log2,
5651 OverloadTys: Builder.getDoubleTy());
5652 llvm::BasicBlock *InputBB = Builder.GetInsertBlock();
5653 llvm::Value *Arg =
5654 Builder.CreateUIToFP(V: ScanRedInfo->Span, DestTy: Builder.getDoubleTy());
5655 llvm::Value *LogVal = emitNoUnwindRuntimeCall(Builder, Callee: F, Args: Arg, Name: "");
5656 F = llvm::Intrinsic::getOrInsertDeclaration(
5657 M: Builder.getModule(), id: (llvm::Intrinsic::ID)llvm::Intrinsic::ceil,
5658 OverloadTys: Builder.getDoubleTy());
5659 LogVal = emitNoUnwindRuntimeCall(Builder, Callee: F, Args: LogVal, Name: "");
5660 LogVal = Builder.CreateFPToUI(V: LogVal, DestTy: Builder.getInt32Ty());
5661 llvm::Value *NMin1 = Builder.CreateNUWSub(
5662 LHS: ScanRedInfo->Span,
5663 RHS: llvm::ConstantInt::get(Ty: ScanRedInfo->Span->getType(), V: 1));
5664 Builder.SetInsertPoint(InputBB);
5665 Builder.CreateBr(Dest: LoopBB);
5666 emitBlock(BB: LoopBB, CurFn);
5667 Builder.SetInsertPoint(LoopBB);
5668
5669 PHINode *Counter = Builder.CreatePHI(Ty: Builder.getInt32Ty(), NumReservedValues: 2);
5670 // size pow2k = 1;
5671 PHINode *Pow2K = Builder.CreatePHI(Ty: Builder.getInt32Ty(), NumReservedValues: 2);
5672 Counter->addIncoming(V: llvm::ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0),
5673 BB: InputBB);
5674 Pow2K->addIncoming(V: llvm::ConstantInt::get(Ty: Builder.getInt32Ty(), V: 1),
5675 BB: InputBB);
5676 // for (size i = n - 1; i >= 2 ^ k; --i)
5677 // tmp[i] op= tmp[i-pow2k];
5678 llvm::BasicBlock *InnerLoopBB =
5679 BasicBlock::Create(Context&: CurFn->getContext(), Name: "omp.inner.log.scan.body");
5680 llvm::BasicBlock *InnerExitBB =
5681 BasicBlock::Create(Context&: CurFn->getContext(), Name: "omp.inner.log.scan.exit");
5682 llvm::Value *CmpI = Builder.CreateICmpUGE(LHS: NMin1, RHS: Pow2K);
5683 Builder.CreateCondBr(Cond: CmpI, True: InnerLoopBB, False: InnerExitBB);
5684 emitBlock(BB: InnerLoopBB, CurFn);
5685 Builder.SetInsertPoint(InnerLoopBB);
5686 PHINode *IVal = Builder.CreatePHI(Ty: Builder.getInt32Ty(), NumReservedValues: 2);
5687 IVal->addIncoming(V: NMin1, BB: LoopBB);
5688 for (ReductionInfo RedInfo : ReductionInfos) {
5689 Value *ReductionVal = RedInfo.PrivateVariable;
5690 Value *BuffPtr = (*(ScanRedInfo->ScanBuffPtrs))[ReductionVal];
5691 Value *Buff = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: BuffPtr);
5692 Type *DestTy = RedInfo.ElementType;
5693 Value *IV = Builder.CreateAdd(LHS: IVal, RHS: Builder.getInt32(C: 1));
5694 Value *LHSPtr =
5695 Builder.CreateInBoundsGEP(Ty: DestTy, Ptr: Buff, IdxList: IV, Name: "arrayOffset");
5696 Value *OffsetIval = Builder.CreateNUWSub(LHS: IV, RHS: Pow2K);
5697 Value *RHSPtr =
5698 Builder.CreateInBoundsGEP(Ty: DestTy, Ptr: Buff, IdxList: OffsetIval, Name: "arrayOffset");
5699 Value *LHS = Builder.CreateLoad(Ty: DestTy, Ptr: LHSPtr);
5700 Value *RHS = Builder.CreateLoad(Ty: DestTy, Ptr: RHSPtr);
5701 llvm::Value *Result;
5702 InsertPointOrErrorTy AfterIP =
5703 RedInfo.ReductionGen(Builder.saveIP(), LHS, RHS, Result);
5704 if (!AfterIP)
5705 return AfterIP.takeError();
5706 Builder.CreateStore(Val: Result, Ptr: LHSPtr);
5707 }
5708 llvm::Value *NextIVal = Builder.CreateNUWSub(
5709 LHS: IVal, RHS: llvm::ConstantInt::get(Ty: Builder.getInt32Ty(), V: 1));
5710 IVal->addIncoming(V: NextIVal, BB: Builder.GetInsertBlock());
5711 CmpI = Builder.CreateICmpUGE(LHS: NextIVal, RHS: Pow2K);
5712 Builder.CreateCondBr(Cond: CmpI, True: InnerLoopBB, False: InnerExitBB);
5713 emitBlock(BB: InnerExitBB, CurFn);
5714 llvm::Value *Next = Builder.CreateNUWAdd(
5715 LHS: Counter, RHS: llvm::ConstantInt::get(Ty: Counter->getType(), V: 1));
5716 Counter->addIncoming(V: Next, BB: Builder.GetInsertBlock());
5717 // pow2k <<= 1;
5718 llvm::Value *NextPow2K = Builder.CreateShl(LHS: Pow2K, RHS: 1, Name: "", /*HasNUW=*/true);
5719 Pow2K->addIncoming(V: NextPow2K, BB: Builder.GetInsertBlock());
5720 llvm::Value *Cmp = Builder.CreateICmpNE(LHS: Next, RHS: LogVal);
5721 Builder.CreateCondBr(Cond: Cmp, True: LoopBB, False: ExitBB);
5722 Builder.SetInsertPoint(ExitBB->getFirstInsertionPt());
5723 return Error::success();
5724 };
5725
5726 // TODO: Perform finalization actions for variables. This has to be
5727 // called for variables which have destructors/finalizers.
5728 auto FiniCB = [&](InsertPointTy CodeGenIP) { return llvm::Error::success(); };
5729
5730 llvm::Value *FilterVal = Builder.getInt32(C: 0);
5731 llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
5732 createMasked(Loc: Builder, BodyGenCB, FiniCB, Filter: FilterVal);
5733
5734 if (!AfterIP)
5735 return AfterIP.takeError();
5736 Builder.restoreIP(IP: *AfterIP);
5737 AfterIP = createBarrier(Loc: Builder, Kind: llvm::omp::OMPD_barrier);
5738
5739 if (!AfterIP)
5740 return AfterIP.takeError();
5741 Builder.restoreIP(IP: *AfterIP);
5742 Error Err = emitScanBasedDirectiveFinalsIR(ReductionInfos, ScanRedInfo);
5743 if (Err)
5744 return Err;
5745
5746 return AfterIP;
5747}
5748
5749Error OpenMPIRBuilder::emitScanBasedDirectiveIR(
5750 llvm::function_ref<Error()> InputLoopGen,
5751 llvm::function_ref<Error(LocationDescription Loc)> ScanLoopGen,
5752 ScanInfo *ScanRedInfo) {
5753
5754 {
5755 // Emit loop with input phase:
5756 // for (i: 0..<num_iters>) {
5757 // <input phase>;
5758 // buffer[i] = red;
5759 // }
5760 ScanRedInfo->OMPFirstScanLoop = true;
5761 Error Err = InputLoopGen();
5762 if (Err)
5763 return Err;
5764 }
5765 {
5766 // Emit loop with scan phase:
5767 // for (i: 0..<num_iters>) {
5768 // red = buffer[i];
5769 // <scan phase>;
5770 // }
5771 ScanRedInfo->OMPFirstScanLoop = false;
5772 Error Err = ScanLoopGen(Builder);
5773 if (Err)
5774 return Err;
5775 }
5776 return Error::success();
5777}
5778
5779void OpenMPIRBuilder::createScanBBs(ScanInfo *ScanRedInfo) {
5780 Function *Fun = Builder.GetInsertBlock()->getParent();
5781 ScanRedInfo->OMPScanDispatch =
5782 BasicBlock::Create(Context&: Fun->getContext(), Name: "omp.inscan.dispatch");
5783 ScanRedInfo->OMPAfterScanBlock =
5784 BasicBlock::Create(Context&: Fun->getContext(), Name: "omp.after.scan.bb");
5785 ScanRedInfo->OMPBeforeScanBlock =
5786 BasicBlock::Create(Context&: Fun->getContext(), Name: "omp.before.scan.bb");
5787 ScanRedInfo->OMPScanLoopExit =
5788 BasicBlock::Create(Context&: Fun->getContext(), Name: "omp.scan.loop.exit");
5789}
5790CanonicalLoopInfo *OpenMPIRBuilder::createLoopSkeleton(
5791 DebugLoc DL, Value *TripCount, Function *F, BasicBlock *PreInsertBefore,
5792 BasicBlock *PostInsertBefore, const Twine &Name, bool IsCollapsed) {
5793 Module *M = F->getParent();
5794 LLVMContext &Ctx = M->getContext();
5795 Type *IndVarTy = TripCount->getType();
5796
5797 // Create the basic block structure.
5798 BasicBlock *Preheader =
5799 BasicBlock::Create(Context&: Ctx, Name: "omp_" + Name + ".preheader", Parent: F, InsertBefore: PreInsertBefore);
5800 BasicBlock *Header =
5801 BasicBlock::Create(Context&: Ctx, Name: "omp_" + Name + ".header", Parent: F, InsertBefore: PreInsertBefore);
5802 BasicBlock *Cond =
5803 BasicBlock::Create(Context&: Ctx, Name: "omp_" + Name + ".cond", Parent: F, InsertBefore: PreInsertBefore);
5804 BasicBlock *Body =
5805 BasicBlock::Create(Context&: Ctx, Name: "omp_" + Name + ".body", Parent: F, InsertBefore: PreInsertBefore);
5806 BasicBlock *Latch =
5807 BasicBlock::Create(Context&: Ctx, Name: "omp_" + Name + ".inc", Parent: F, InsertBefore: PostInsertBefore);
5808 BasicBlock *Exit =
5809 BasicBlock::Create(Context&: Ctx, Name: "omp_" + Name + ".exit", Parent: F, InsertBefore: PostInsertBefore);
5810 BasicBlock *After =
5811 BasicBlock::Create(Context&: Ctx, Name: "omp_" + Name + ".after", Parent: F, InsertBefore: PostInsertBefore);
5812
5813 // Use specified DebugLoc for new instructions.
5814 Builder.SetCurrentDebugLocation(DL);
5815
5816 Builder.SetInsertPoint(Preheader);
5817 Builder.CreateBr(Dest: Header);
5818
5819 Builder.SetInsertPoint(Header);
5820 PHINode *IndVarPHI = Builder.CreatePHI(Ty: IndVarTy, NumReservedValues: 2, Name: "omp_" + Name + ".iv");
5821 IndVarPHI->addIncoming(V: ConstantInt::get(Ty: IndVarTy, V: 0), BB: Preheader);
5822 Builder.CreateBr(Dest: Cond);
5823
5824 Builder.SetInsertPoint(Cond);
5825 Value *Cmp =
5826 Builder.CreateICmpULT(LHS: IndVarPHI, RHS: TripCount, Name: "omp_" + Name + ".cmp");
5827 Builder.CreateCondBr(Cond: Cmp, True: Body, False: Exit);
5828
5829 Builder.SetInsertPoint(Body);
5830 Builder.CreateBr(Dest: Latch);
5831
5832 Builder.SetInsertPoint(Latch);
5833 // Decide whether the induction variable increment can carry nsw.
5834 //
5835 // Single loops: nsw is always kept (matching Clang). Any Fortran program
5836 // whose trip count overflows i32 is non-conforming per F2018 11.1.7.4.1, so
5837 // for valid programs 0 <= count <= INT_MAX always holds.
5838 //
5839 // Collapsed loops: the trip count is a product that can overflow i32 even for
5840 // a conforming program, so nsw is kept only when the product is a constant
5841 // that provably fits, dropped otherwise.
5842 bool HasNSW = Config.hasNoSignedWrap();
5843 if (HasNSW) {
5844 if (auto *CI = dyn_cast<ConstantInt>(Val: TripCount)) {
5845 unsigned BitWidth = CI->getType()->getIntegerBitWidth();
5846 APInt SignedMax = APInt::getSignedMaxValue(numBits: BitWidth);
5847 if (CI->getValue().ugt(RHS: SignedMax))
5848 HasNSW = false;
5849 } else if (IsCollapsed) {
5850 HasNSW = false;
5851 }
5852 }
5853 Value *Next =
5854 Builder.CreateAdd(LHS: IndVarPHI, RHS: ConstantInt::get(Ty: IndVarTy, V: 1),
5855 Name: "omp_" + Name + ".next", /*HasNUW=*/true, HasNSW);
5856 Builder.CreateBr(Dest: Header);
5857 IndVarPHI->addIncoming(V: Next, BB: Latch);
5858
5859 Builder.SetInsertPoint(Exit);
5860 Builder.CreateBr(Dest: After);
5861
5862 // Remember and return the canonical control flow.
5863 LoopInfos.emplace_front();
5864 CanonicalLoopInfo *CL = &LoopInfos.front();
5865
5866 CL->Header = Header;
5867 CL->Cond = Cond;
5868 CL->Latch = Latch;
5869 CL->Exit = Exit;
5870
5871#ifndef NDEBUG
5872 CL->assertOK();
5873#endif
5874 return CL;
5875}
5876
5877Expected<CanonicalLoopInfo *>
5878OpenMPIRBuilder::createCanonicalLoop(const LocationDescription &Loc,
5879 LoopBodyGenCallbackTy BodyGenCB,
5880 Value *TripCount, const Twine &Name) {
5881 BasicBlock *BB = Loc.IP.getNodeParent();
5882 BasicBlock *NextBB = BB->getNextNode();
5883
5884 CanonicalLoopInfo *CL = createLoopSkeleton(DL: Loc.DL, TripCount, F: BB->getParent(),
5885 PreInsertBefore: NextBB, PostInsertBefore: NextBB, Name);
5886 BasicBlock *After = CL->getAfter();
5887
5888 // If location is not set, don't connect the loop.
5889 if (updateToLocation(Loc)) {
5890 // Split the loop at the insertion point: Branch to the preheader and move
5891 // every following instruction to after the loop (the After BB). Also, the
5892 // new successor is the loop's after block.
5893 spliceBB(Builder, New: After, /*CreateBranch=*/false);
5894 Builder.CreateBr(Dest: CL->getPreheader());
5895 }
5896
5897 // Emit the body content. We do it after connecting the loop to the CFG to
5898 // avoid that the callback encounters degenerate BBs.
5899 if (Error Err = BodyGenCB(CL->getBodyIP(), CL->getIndVar()))
5900 return Err;
5901
5902#ifndef NDEBUG
5903 CL->assertOK();
5904#endif
5905 return CL;
5906}
5907
5908Expected<ScanInfo *> OpenMPIRBuilder::scanInfoInitialize() {
5909 ScanInfos.emplace_front();
5910 ScanInfo *Result = &ScanInfos.front();
5911 return Result;
5912}
5913
5914Expected<SmallVector<llvm::CanonicalLoopInfo *>>
5915OpenMPIRBuilder::createCanonicalScanLoops(
5916 const LocationDescription &Loc, LoopBodyGenCallbackTy BodyGenCB,
5917 Value *Start, Value *Stop, Value *Step, bool IsSigned, bool InclusiveStop,
5918 InsertPointTy ComputeIP, const Twine &Name, ScanInfo *ScanRedInfo) {
5919 LocationDescription ComputeLoc =
5920 ComputeIP.isValid() ? LocationDescription(ComputeIP, Loc.DL) : Loc;
5921 updateToLocation(Loc: ComputeLoc);
5922
5923 SmallVector<CanonicalLoopInfo *> Result;
5924
5925 Value *TripCount = calculateCanonicalLoopTripCount(
5926 Loc: ComputeLoc, Start, Stop, Step, IsSigned, InclusiveStop, Name);
5927 ScanRedInfo->Span = TripCount;
5928 ScanRedInfo->OMPScanInit = splitBB(Builder, CreateBranch: true, Name: "scan.init");
5929 Builder.SetInsertPoint(ScanRedInfo->OMPScanInit);
5930
5931 auto BodyGen = [=](InsertPointTy CodeGenIP, Value *IV) {
5932 Builder.restoreIP(IP: CodeGenIP);
5933 ScanRedInfo->IV = IV;
5934 createScanBBs(ScanRedInfo);
5935 BasicBlock *InputBlock = Builder.GetInsertBlock();
5936 Instruction *Terminator = InputBlock->getTerminator();
5937 assert(Terminator->getNumSuccessors() == 1);
5938 BasicBlock *ContinueBlock = Terminator->getSuccessor(Idx: 0);
5939 Terminator->setSuccessor(Idx: 0, BB: ScanRedInfo->OMPScanDispatch);
5940 emitBlock(BB: ScanRedInfo->OMPBeforeScanBlock,
5941 CurFn: Builder.GetInsertBlock()->getParent());
5942 Builder.CreateBr(Dest: ScanRedInfo->OMPScanLoopExit);
5943 emitBlock(BB: ScanRedInfo->OMPScanLoopExit,
5944 CurFn: Builder.GetInsertBlock()->getParent());
5945 Builder.CreateBr(Dest: ContinueBlock);
5946 Builder.SetInsertPoint(
5947 ScanRedInfo->OMPBeforeScanBlock->getFirstInsertionPt());
5948 return BodyGenCB(Builder.saveIP(), IV);
5949 };
5950
5951 const auto &&InputLoopGen = [&]() -> Error {
5952 Expected<CanonicalLoopInfo *> LoopInfo =
5953 createCanonicalLoop(Loc: Builder, BodyGenCB: BodyGen, Start, Stop, Step, IsSigned,
5954 InclusiveStop, ComputeIP, Name, InScan: true, ScanRedInfo);
5955 if (!LoopInfo)
5956 return LoopInfo.takeError();
5957 Result.push_back(Elt: *LoopInfo);
5958 Builder.restoreIP(IP: (*LoopInfo)->getAfterIP());
5959 return Error::success();
5960 };
5961 const auto &&ScanLoopGen = [&](LocationDescription Loc) -> Error {
5962 Expected<CanonicalLoopInfo *> LoopInfo =
5963 createCanonicalLoop(Loc, BodyGenCB: BodyGen, Start, Stop, Step, IsSigned,
5964 InclusiveStop, ComputeIP, Name, InScan: true, ScanRedInfo);
5965 if (!LoopInfo)
5966 return LoopInfo.takeError();
5967 Result.push_back(Elt: *LoopInfo);
5968 Builder.restoreIP(IP: (*LoopInfo)->getAfterIP());
5969 ScanRedInfo->OMPScanFinish = Builder.GetInsertBlock();
5970 return Error::success();
5971 };
5972 Error Err = emitScanBasedDirectiveIR(InputLoopGen, ScanLoopGen, ScanRedInfo);
5973 if (Err)
5974 return Err;
5975 return Result;
5976}
5977
5978Value *OpenMPIRBuilder::calculateCanonicalLoopTripCount(
5979 const LocationDescription &Loc, Value *Start, Value *Stop, Value *Step,
5980 bool IsSigned, bool InclusiveStop, const Twine &Name) {
5981
5982 // Consider the following difficulties (assuming 8-bit signed integers):
5983 // * Adding \p Step to the loop counter which passes \p Stop may overflow:
5984 // DO I = 1, 100, 50
5985 /// * A \p Step of INT_MIN cannot not be normalized to a positive direction:
5986 // DO I = 100, 0, -128
5987
5988 // Start, Stop and Step must be of the same integer type.
5989 auto *IndVarTy = cast<IntegerType>(Val: Start->getType());
5990 assert(IndVarTy == Stop->getType() && "Stop type mismatch");
5991 assert(IndVarTy == Step->getType() && "Step type mismatch");
5992
5993 updateToLocation(Loc);
5994
5995 ConstantInt *Zero = ConstantInt::get(Ty: IndVarTy, V: 0);
5996 ConstantInt *One = ConstantInt::get(Ty: IndVarTy, V: 1);
5997
5998 // Like Step, but always positive.
5999 Value *Incr = Step;
6000
6001 // Distance between Start and Stop; always positive.
6002 Value *Span;
6003
6004 // Condition whether there are no iterations are executed at all, e.g. because
6005 // UB < LB.
6006 Value *ZeroCmp;
6007
6008 if (IsSigned) {
6009 // Ensure that increment is positive. If not, negate and invert LB and UB.
6010 Value *IsNeg = Builder.CreateICmpSLT(LHS: Step, RHS: Zero);
6011 Incr = Builder.CreateSelect(C: IsNeg, True: Builder.CreateNeg(V: Step), False: Step);
6012 Value *LB = Builder.CreateSelect(C: IsNeg, True: Stop, False: Start);
6013 Value *UB = Builder.CreateSelect(C: IsNeg, True: Start, False: Stop);
6014 Span = Builder.CreateSub(LHS: UB, RHS: LB, Name: "", HasNUW: false, HasNSW: true);
6015 ZeroCmp = Builder.CreateICmp(
6016 P: InclusiveStop ? CmpInst::ICMP_SLT : CmpInst::ICMP_SLE, LHS: UB, RHS: LB);
6017 } else {
6018 Span = Builder.CreateSub(LHS: Stop, RHS: Start, Name: "", HasNUW: true);
6019 ZeroCmp = Builder.CreateICmp(
6020 P: InclusiveStop ? CmpInst::ICMP_ULT : CmpInst::ICMP_ULE, LHS: Stop, RHS: Start);
6021 }
6022
6023 Value *CountIfLooping;
6024 if (InclusiveStop) {
6025 CountIfLooping = Builder.CreateAdd(LHS: Builder.CreateUDiv(LHS: Span, RHS: Incr), RHS: One);
6026 } else {
6027 // Avoid incrementing past stop since it could overflow.
6028 Value *CountIfTwo = Builder.CreateAdd(
6029 LHS: Builder.CreateUDiv(LHS: Builder.CreateSub(LHS: Span, RHS: One), RHS: Incr), RHS: One);
6030 Value *OneCmp = Builder.CreateICmp(P: CmpInst::ICMP_ULE, LHS: Span, RHS: Incr);
6031 CountIfLooping = Builder.CreateSelect(C: OneCmp, True: One, False: CountIfTwo);
6032 }
6033
6034 return Builder.CreateSelect(C: ZeroCmp, True: Zero, False: CountIfLooping,
6035 Name: "omp_" + Name + ".tripcount");
6036}
6037
6038Expected<CanonicalLoopInfo *> OpenMPIRBuilder::createCanonicalLoop(
6039 const LocationDescription &Loc, LoopBodyGenCallbackTy BodyGenCB,
6040 Value *Start, Value *Stop, Value *Step, bool IsSigned, bool InclusiveStop,
6041 InsertPointTy ComputeIP, const Twine &Name, bool InScan,
6042 ScanInfo *ScanRedInfo) {
6043 LocationDescription ComputeLoc =
6044 ComputeIP.isValid() ? LocationDescription(ComputeIP, Loc.DL) : Loc;
6045
6046 Value *TripCount = calculateCanonicalLoopTripCount(
6047 Loc: ComputeLoc, Start, Stop, Step, IsSigned, InclusiveStop, Name);
6048
6049 auto BodyGen = [=](InsertPointTy CodeGenIP, Value *IV) {
6050 Builder.restoreIP(IP: CodeGenIP);
6051 Value *Span = Builder.CreateMul(LHS: IV, RHS: Step, Name: "", /*HasNUW=*/false,
6052 /*HasNSW=*/Config.hasNoSignedWrap());
6053 Value *IndVar = Builder.CreateAdd(LHS: Span, RHS: Start, Name: "", /*HasNUW=*/false,
6054 /*HasNSW=*/Config.hasNoSignedWrap());
6055 if (InScan)
6056 ScanRedInfo->IV = IndVar;
6057 return BodyGenCB(Builder.saveIP(), IndVar);
6058 };
6059 LocationDescription LoopLoc =
6060 ComputeIP.isValid()
6061 ? Loc
6062 : LocationDescription(Builder.saveIP(),
6063 Builder.getCurrentDebugLocation());
6064 return createCanonicalLoop(Loc: LoopLoc, BodyGenCB: BodyGen, TripCount, Name);
6065}
6066
6067// Returns an LLVM function to call for initializing loop bounds using OpenMP
6068// static scheduling for composite `distribute parallel for` depending on
6069// `type`. Only i32 and i64 are supported by the runtime. Always interpret
6070// integers as unsigned similarly to CanonicalLoopInfo.
6071static FunctionCallee
6072getKmpcDistForStaticInitForType(Type *Ty, Module &M,
6073 OpenMPIRBuilder &OMPBuilder) {
6074 unsigned Bitwidth = Ty->getIntegerBitWidth();
6075 if (Bitwidth == 32)
6076 return OMPBuilder.getOrCreateRuntimeFunction(
6077 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dist_for_static_init_4u);
6078 if (Bitwidth == 64)
6079 return OMPBuilder.getOrCreateRuntimeFunction(
6080 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dist_for_static_init_8u);
6081 llvm_unreachable("unknown OpenMP loop iterator bitwidth");
6082}
6083
6084// Returns an LLVM function to call for initializing loop bounds using OpenMP
6085// static scheduling depending on `type`. Only i32 and i64 are supported by the
6086// runtime. Always interpret integers as unsigned similarly to
6087// CanonicalLoopInfo.
6088static FunctionCallee getKmpcForStaticInitForType(Type *Ty, Module &M,
6089 OpenMPIRBuilder &OMPBuilder) {
6090 unsigned Bitwidth = Ty->getIntegerBitWidth();
6091 if (Bitwidth == 32)
6092 return OMPBuilder.getOrCreateRuntimeFunction(
6093 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_4u);
6094 if (Bitwidth == 64)
6095 return OMPBuilder.getOrCreateRuntimeFunction(
6096 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_8u);
6097 llvm_unreachable("unknown OpenMP loop iterator bitwidth");
6098}
6099
6100OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::applyStaticWorkshareLoop(
6101 DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP,
6102 WorksharingLoopType LoopType, bool NeedsBarrier, bool HasDistSchedule,
6103 OMPScheduleType DistScheduleSchedType) {
6104 assert(CLI->isValid() && "Requires a valid canonical loop");
6105 assert(!isConflictIP(AllocaIP, CLI->getPreheaderIP()) &&
6106 "Require dedicated allocate IP");
6107
6108 // Set up the source location value for OpenMP runtime.
6109 Builder.restoreIP(IP: CLI->getPreheaderIP());
6110 Builder.SetCurrentDebugLocation(DL);
6111
6112 uint32_t SrcLocStrSize;
6113 Constant *SrcLocStr = getOrCreateSrcLocStr(DL, SrcLocStrSize);
6114 IdentFlag Flag = IdentFlag(0);
6115 switch (LoopType) {
6116 case WorksharingLoopType::ForStaticLoop:
6117 Flag = OMP_IDENT_FLAG_WORK_LOOP;
6118 break;
6119 case WorksharingLoopType::DistributeStaticLoop:
6120 Flag = OMP_IDENT_FLAG_WORK_DISTRIBUTE;
6121 break;
6122 case WorksharingLoopType::DistributeForStaticLoop:
6123 Flag = OMP_IDENT_FLAG_WORK_DISTRIBUTE | OMP_IDENT_FLAG_WORK_LOOP;
6124 break;
6125 }
6126 Value *SrcLoc = getOrCreateIdent(SrcLocStr, SrcLocStrSize, LocFlags: Flag);
6127
6128 // Declare useful OpenMP runtime functions.
6129 Type *IVTy = CLI->getIndVarType();
6130 FunctionCallee StaticInit =
6131 LoopType == WorksharingLoopType::DistributeForStaticLoop
6132 ? getKmpcDistForStaticInitForType(Ty: IVTy, M, OMPBuilder&: *this)
6133 : getKmpcForStaticInitForType(Ty: IVTy, M, OMPBuilder&: *this);
6134 FunctionCallee StaticFini =
6135 getOrCreateRuntimeFunction(M, FnID: omp::OMPRTL___kmpc_for_static_fini);
6136
6137 // Allocate space for computed loop bounds as expected by the "init" function.
6138 Builder.SetInsertPoint(AllocaIP.getNodeParent()->getFirstNonPHIOrAlloca());
6139
6140 Type *I32Type = Type::getInt32Ty(C&: M.getContext());
6141 Value *PLastIter = Builder.CreateAlloca(Ty: I32Type, ArraySize: nullptr, Name: "p.lastiter");
6142 Value *PLowerBound = Builder.CreateAlloca(Ty: IVTy, ArraySize: nullptr, Name: "p.lowerbound");
6143 Value *PUpperBound = Builder.CreateAlloca(Ty: IVTy, ArraySize: nullptr, Name: "p.upperbound");
6144 Value *PStride = Builder.CreateAlloca(Ty: IVTy, ArraySize: nullptr, Name: "p.stride");
6145 CLI->setLastIter(PLastIter);
6146
6147 // At the end of the preheader, prepare for calling the "init" function by
6148 // storing the current loop bounds into the allocated space. A canonical loop
6149 // always iterates from 0 to trip-count with step 1. Note that "init" expects
6150 // and produces an inclusive upper bound.
6151 Builder.SetInsertPoint(CLI->getPreheader()->getTerminator());
6152 Constant *Zero = ConstantInt::get(Ty: IVTy, V: 0);
6153 Constant *One = ConstantInt::get(Ty: IVTy, V: 1);
6154 Builder.CreateStore(Val: Zero, Ptr: PLowerBound);
6155 Value *UpperBound = Builder.CreateSub(LHS: CLI->getTripCount(), RHS: One);
6156 Builder.CreateStore(Val: UpperBound, Ptr: PUpperBound);
6157 Builder.CreateStore(Val: One, Ptr: PStride);
6158
6159 Value *ThreadNum =
6160 getOrCreateThreadID(Ident: getOrCreateIdent(SrcLocStr, SrcLocStrSize));
6161
6162 OMPScheduleType SchedType =
6163 (LoopType == WorksharingLoopType::DistributeStaticLoop)
6164 ? OMPScheduleType::OrderedDistribute
6165 : OMPScheduleType::UnorderedStatic;
6166 Constant *SchedulingType =
6167 ConstantInt::get(Ty: I32Type, V: static_cast<int>(SchedType));
6168
6169 // Call the "init" function and update the trip count of the loop with the
6170 // value it produced.
6171 auto BuildInitCall = [LoopType, SrcLoc, ThreadNum, PLastIter, PLowerBound,
6172 PUpperBound, IVTy, PStride, One, Zero, StaticInit,
6173 this](Value *SchedulingType, auto &Builder) {
6174 SmallVector<Value *, 10> Args({SrcLoc, ThreadNum, SchedulingType, PLastIter,
6175 PLowerBound, PUpperBound});
6176 if (LoopType == WorksharingLoopType::DistributeForStaticLoop) {
6177 Value *PDistUpperBound =
6178 Builder.CreateAlloca(IVTy, nullptr, "p.distupperbound");
6179 Args.push_back(Elt: PDistUpperBound);
6180 }
6181 Args.append(IL: {PStride, One, Zero});
6182 createRuntimeFunctionCall(Callee: StaticInit, Args);
6183 };
6184 BuildInitCall(SchedulingType, Builder);
6185 if (HasDistSchedule &&
6186 LoopType != WorksharingLoopType::DistributeStaticLoop) {
6187 Constant *DistScheduleSchedType = ConstantInt::get(
6188 Ty: I32Type, V: static_cast<int>(omp::OMPScheduleType::OrderedDistribute));
6189 // We want to emit a second init function call for the dist_schedule clause
6190 // to the Distribute construct. This should only be done however if a
6191 // Workshare Loop is nested within a Distribute Construct
6192 BuildInitCall(DistScheduleSchedType, Builder);
6193 }
6194 Value *LowerBound = Builder.CreateLoad(Ty: IVTy, Ptr: PLowerBound);
6195 Value *InclusiveUpperBound = Builder.CreateLoad(Ty: IVTy, Ptr: PUpperBound);
6196 Value *TripCountMinusOne = Builder.CreateSub(LHS: InclusiveUpperBound, RHS: LowerBound);
6197 Value *TripCount = Builder.CreateAdd(LHS: TripCountMinusOne, RHS: One);
6198 if (CLI->VirtualCollapse) {
6199 bindCollapsedRange(Loop: CLI, Begin: LowerBound, Count: TripCount);
6200#ifndef NDEBUG
6201 CLI->assertOK();
6202#endif
6203 } else {
6204 CLI->setTripCount(TripCount);
6205 }
6206
6207 // Update all uses of the induction variable except the one in the condition
6208 // block that compares it with the actual upper bound, and the increment in
6209 // the latch block.
6210
6211 if (!CLI->VirtualCollapse)
6212 CLI->mapIndVar(Updater: [&](Instruction *OldIV) -> Value * {
6213 Builder.SetInsertPoint(CLI->getBody()->getFirstInsertionPt());
6214 Builder.SetCurrentDebugLocation(DL);
6215 return Builder.CreateAdd(LHS: OldIV, RHS: LowerBound, Name: "", /*HasNUW=*/false,
6216 /*HasNSW=*/Config.hasNoSignedWrap());
6217 });
6218
6219 // In the "exit" block, call the "fini" function.
6220 Builder.SetInsertPoint(CLI->getExit()->getTerminator()->getIterator());
6221 createRuntimeFunctionCall(Callee: StaticFini, Args: {SrcLoc, ThreadNum});
6222
6223 // Add the barrier if requested.
6224 if (NeedsBarrier) {
6225 InsertPointOrErrorTy BarrierIP =
6226 createBarrier(Loc: LocationDescription(Builder.saveIP(), DL),
6227 Kind: omp::Directive::OMPD_for, /* ForceSimpleCall */ false,
6228 /* CheckCancelFlag */ false);
6229 if (!BarrierIP)
6230 return BarrierIP.takeError();
6231 }
6232
6233 InsertPointTy AfterIP = CLI->getAfterIP();
6234 // TODO: Removing Virtual's outer bound requires another way to reject idle
6235 // assignments whose raw count wraps. Keep it for correctness and trip-count
6236 // information for analyses. The expected saving is small; investigate the
6237 // effects on downstream optimizations before removing it.
6238 CLI->invalidate();
6239
6240 return AfterIP;
6241}
6242
6243static void addAccessGroupMetadata(BasicBlock *Block, MDNode *AccessGroup,
6244 LoopInfo &LI);
6245static void addLoopMetadata(CanonicalLoopInfo *Loop,
6246 ArrayRef<Metadata *> Properties);
6247
6248static void applyParallelAccessesMetadata(CanonicalLoopInfo *CLI,
6249 LLVMContext &Ctx, Loop *Loop,
6250 LoopInfo &LoopInfo,
6251 SmallVector<Metadata *> &LoopMDList) {
6252 SmallSet<BasicBlock *, 8> Reachable;
6253
6254 // Get the basic blocks from the loop in which memref instructions
6255 // can be found.
6256 // TODO: Generalize getting all blocks inside a CanonicalizeLoopInfo,
6257 // preferably without running any passes.
6258 for (BasicBlock *Block : Loop->getBlocks()) {
6259 if (Block == CLI->getCond() || Block == CLI->getHeader())
6260 continue;
6261 Reachable.insert(Ptr: Block);
6262 }
6263
6264 // Add access group metadata to memory-access instructions.
6265 MDNode *AccessGroup = MDNode::getDistinct(Context&: Ctx, MDs: {});
6266 for (BasicBlock *BB : Reachable)
6267 addAccessGroupMetadata(Block: BB, AccessGroup, LI&: LoopInfo);
6268 // TODO: If the loop has existing parallel access metadata, have
6269 // to combine two lists.
6270 LoopMDList.push_back(Elt: MDNode::get(
6271 Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.parallel_accesses"), AccessGroup}));
6272}
6273
6274OpenMPIRBuilder::InsertPointOrErrorTy
6275OpenMPIRBuilder::applyStaticChunkedWorkshareLoop(
6276 DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP,
6277 bool NeedsBarrier, Value *ChunkSize, OMPScheduleType SchedType,
6278 Value *DistScheduleChunkSize, OMPScheduleType DistScheduleSchedType) {
6279 assert(CLI->isValid() && "Requires a valid canonical loop");
6280 assert((ChunkSize || DistScheduleChunkSize) && "Chunk size is required");
6281
6282 LLVMContext &Ctx = CLI->getFunction()->getContext();
6283 Value *IV = CLI->getIndVar();
6284 Value *OrigTripCount = CLI->getTripCount();
6285 Type *IVTy = IV->getType();
6286 assert(IVTy->getIntegerBitWidth() <= 64 &&
6287 "Max supported tripcount bitwidth is 64 bits");
6288 Type *InternalIVTy = IVTy->getIntegerBitWidth() <= 32 ? Type::getInt32Ty(C&: Ctx)
6289 : Type::getInt64Ty(C&: Ctx);
6290 Type *I32Type = Type::getInt32Ty(C&: M.getContext());
6291 Constant *Zero = ConstantInt::get(Ty: InternalIVTy, V: 0);
6292 Constant *One = ConstantInt::get(Ty: InternalIVTy, V: 1);
6293
6294 Function *F = CLI->getFunction();
6295 // Blocks must have terminators.
6296 // FIXME: Don't run analyses on incomplete/invalid IR.
6297 SmallVector<Instruction *> UIs;
6298 for (BasicBlock &BB : *F)
6299 if (!BB.hasTerminator())
6300 UIs.push_back(Elt: new UnreachableInst(F->getContext(), &BB));
6301 FunctionAnalysisManager FAM;
6302 FAM.registerPass(PassBuilder: []() { return DominatorTreeAnalysis(); });
6303 FAM.registerPass(PassBuilder: []() { return PassInstrumentationAnalysis(); });
6304 LoopAnalysis LIA;
6305 LoopInfo &&LI = LIA.run(F&: *F, AM&: FAM);
6306 for (Instruction *I : UIs)
6307 I->eraseFromParent();
6308 Loop *L = LI.getLoopFor(BB: CLI->getHeader());
6309 SmallVector<Metadata *> LoopMDList;
6310 if (ChunkSize || DistScheduleChunkSize)
6311 applyParallelAccessesMetadata(CLI, Ctx, Loop: L, LoopInfo&: LI, LoopMDList);
6312 addLoopMetadata(Loop: CLI, Properties: LoopMDList);
6313
6314 // Declare useful OpenMP runtime functions.
6315 FunctionCallee StaticInit =
6316 getKmpcForStaticInitForType(Ty: InternalIVTy, M, OMPBuilder&: *this);
6317 FunctionCallee StaticFini =
6318 getOrCreateRuntimeFunction(M, FnID: omp::OMPRTL___kmpc_for_static_fini);
6319
6320 // Allocate space for computed loop bounds as expected by the "init" function.
6321 Builder.restoreIP(IP: AllocaIP);
6322 Builder.SetCurrentDebugLocation(DL);
6323 Value *PLastIter = Builder.CreateAlloca(Ty: I32Type, ArraySize: nullptr, Name: "p.lastiter");
6324 Value *PLowerBound =
6325 Builder.CreateAlloca(Ty: InternalIVTy, ArraySize: nullptr, Name: "p.lowerbound");
6326 Value *PUpperBound =
6327 Builder.CreateAlloca(Ty: InternalIVTy, ArraySize: nullptr, Name: "p.upperbound");
6328 Value *PStride = Builder.CreateAlloca(Ty: InternalIVTy, ArraySize: nullptr, Name: "p.stride");
6329 CLI->setLastIter(PLastIter);
6330
6331 // Set up the source location value for the OpenMP runtime.
6332 Builder.restoreIP(IP: CLI->getPreheaderIP());
6333 Builder.SetCurrentDebugLocation(DL);
6334
6335 // TODO: Detect overflow in ubsan or max-out with current tripcount.
6336 Value *CastedChunkSize = Builder.CreateZExtOrTrunc(
6337 V: ChunkSize ? ChunkSize : Zero, DestTy: InternalIVTy, Name: "chunksize");
6338 Value *CastedDistScheduleChunkSize = Builder.CreateZExtOrTrunc(
6339 V: DistScheduleChunkSize ? DistScheduleChunkSize : Zero, DestTy: InternalIVTy,
6340 Name: "distschedulechunksize");
6341 Value *CastedTripCount =
6342 Builder.CreateZExt(V: OrigTripCount, DestTy: InternalIVTy, Name: "tripcount");
6343
6344 Constant *SchedulingType =
6345 ConstantInt::get(Ty: I32Type, V: static_cast<int>(SchedType));
6346 Constant *DistSchedulingType =
6347 ConstantInt::get(Ty: I32Type, V: static_cast<int>(DistScheduleSchedType));
6348 Builder.CreateStore(Val: Zero, Ptr: PLowerBound);
6349 Value *OrigUpperBound = Builder.CreateSub(LHS: CastedTripCount, RHS: One);
6350 Value *IsTripCountZero = Builder.CreateICmpEQ(LHS: CastedTripCount, RHS: Zero);
6351 Value *UpperBound =
6352 Builder.CreateSelect(C: IsTripCountZero, True: Zero, False: OrigUpperBound);
6353 Builder.CreateStore(Val: UpperBound, Ptr: PUpperBound);
6354 Builder.CreateStore(Val: One, Ptr: PStride);
6355
6356 // Call the "init" function and update the trip count of the loop with the
6357 // value it produced.
6358 uint32_t SrcLocStrSize;
6359 Constant *SrcLocStr = getOrCreateSrcLocStr(DL, SrcLocStrSize);
6360 IdentFlag Flag = OMP_IDENT_FLAG_WORK_LOOP;
6361 if (DistScheduleSchedType != OMPScheduleType::None) {
6362 Flag |= OMP_IDENT_FLAG_WORK_DISTRIBUTE;
6363 }
6364 Value *SrcLoc = getOrCreateIdent(SrcLocStr, SrcLocStrSize, LocFlags: Flag);
6365 Value *ThreadNum =
6366 getOrCreateThreadID(Ident: getOrCreateIdent(SrcLocStr, SrcLocStrSize));
6367 auto BuildInitCall = [StaticInit, SrcLoc, ThreadNum, PLastIter, PLowerBound,
6368 PUpperBound, PStride, One,
6369 this](Value *SchedulingType, Value *ChunkSize,
6370 auto &Builder) {
6371 createRuntimeFunctionCall(
6372 Callee: StaticInit, Args: {/*loc=*/SrcLoc, /*global_tid=*/ThreadNum,
6373 /*schedtype=*/SchedulingType, /*plastiter=*/PLastIter,
6374 /*plower=*/PLowerBound, /*pupper=*/PUpperBound,
6375 /*pstride=*/PStride, /*incr=*/One,
6376 /*chunk=*/ChunkSize});
6377 };
6378 BuildInitCall(SchedulingType, CastedChunkSize, Builder);
6379 if (DistScheduleSchedType != OMPScheduleType::None &&
6380 SchedType != OMPScheduleType::OrderedDistributeChunked &&
6381 SchedType != OMPScheduleType::OrderedDistribute) {
6382 // We want to emit a second init function call for the dist_schedule clause
6383 // to the Distribute construct. This should only be done however if a
6384 // Workshare Loop is nested within a Distribute Construct
6385 BuildInitCall(DistSchedulingType, CastedDistScheduleChunkSize, Builder);
6386 }
6387
6388 // Load values written by the "init" function.
6389 Value *FirstChunkStart =
6390 Builder.CreateLoad(Ty: InternalIVTy, Ptr: PLowerBound, Name: "omp_firstchunk.lb");
6391 Value *FirstChunkStop =
6392 Builder.CreateLoad(Ty: InternalIVTy, Ptr: PUpperBound, Name: "omp_firstchunk.ub");
6393 Value *FirstChunkEnd = Builder.CreateAdd(LHS: FirstChunkStop, RHS: One);
6394 Value *ChunkRange =
6395 Builder.CreateSub(LHS: FirstChunkEnd, RHS: FirstChunkStart, Name: "omp_chunk.range");
6396 Value *NextChunkStride =
6397 Builder.CreateLoad(Ty: InternalIVTy, Ptr: PStride, Name: "omp_dispatch.stride");
6398
6399 // Create outer "dispatch" loop for enumerating the chunks.
6400 BasicBlock *DispatchEnter = splitBB(Builder, CreateBranch: true);
6401 Value *DispatchCounter;
6402
6403 // It is safe to assume this didn't return an error because the callback
6404 // passed into createCanonicalLoop is the only possible error source, and it
6405 // always returns success.
6406 CanonicalLoopInfo *DispatchCLI = cantFail(ValOrErr: createCanonicalLoop(
6407 Loc: {Builder.saveIP(), DL},
6408 BodyGenCB: [&](InsertPointTy BodyIP, Value *Counter) {
6409 DispatchCounter = Counter;
6410 return Error::success();
6411 },
6412 Start: FirstChunkStart, Stop: CastedTripCount, Step: NextChunkStride,
6413 /*IsSigned=*/false, /*InclusiveStop=*/false, /*ComputeIP=*/{},
6414 Name: "dispatch"));
6415
6416 // Remember the BasicBlocks of the dispatch loop we need, then invalidate to
6417 // not have to preserve the canonical invariant.
6418 BasicBlock *DispatchBody = DispatchCLI->getBody();
6419 BasicBlock *DispatchLatch = DispatchCLI->getLatch();
6420 BasicBlock *DispatchExit = DispatchCLI->getExit();
6421 BasicBlock *DispatchAfter = DispatchCLI->getAfter();
6422 DispatchCLI->invalidate();
6423
6424 // Rewire the original loop to become the chunk loop inside the dispatch loop.
6425 redirectTo(Source: DispatchAfter, Target: CLI->getAfter(), DL);
6426 redirectTo(Source: CLI->getExit(), Target: DispatchLatch, DL);
6427 redirectTo(Source: DispatchBody, Target: DispatchEnter, DL);
6428
6429 // Prepare the prolog of the chunk loop.
6430 Builder.restoreIP(IP: CLI->getPreheaderIP());
6431 Builder.SetCurrentDebugLocation(DL);
6432
6433 // Compute the number of iterations of the chunk loop.
6434 Builder.SetInsertPoint(CLI->getPreheader()->getTerminator());
6435 Value *ChunkEnd = Builder.CreateAdd(LHS: DispatchCounter, RHS: ChunkRange);
6436 Value *IsLastChunk =
6437 Builder.CreateICmpUGE(LHS: ChunkEnd, RHS: CastedTripCount, Name: "omp_chunk.is_last");
6438 Value *CountUntilOrigTripCount =
6439 Builder.CreateSub(LHS: CastedTripCount, RHS: DispatchCounter);
6440 Value *ChunkTripCount = Builder.CreateSelect(
6441 C: IsLastChunk, True: CountUntilOrigTripCount, False: ChunkRange, Name: "omp_chunk.tripcount");
6442 Value *BackcastedChunkTC =
6443 Builder.CreateTrunc(V: ChunkTripCount, DestTy: IVTy, Name: "omp_chunk.tripcount.trunc");
6444 CLI->setTripCount(BackcastedChunkTC);
6445
6446 // Update all uses of the induction variable except the one in the condition
6447 // block that compares it with the actual upper bound, and the increment in
6448 // the latch block.
6449 Value *BackcastedDispatchCounter =
6450 Builder.CreateTrunc(V: DispatchCounter, DestTy: IVTy, Name: "omp_dispatch.iv.trunc");
6451 CLI->mapIndVar(Updater: [&](Instruction *) -> Value * {
6452 Builder.restoreIP(IP: CLI->getBodyIP());
6453 return Builder.CreateAdd(LHS: IV, RHS: BackcastedDispatchCounter);
6454 });
6455
6456 // In the "exit" block, call the "fini" function.
6457 Builder.SetInsertPoint(DispatchExit->getFirstInsertionPt());
6458 createRuntimeFunctionCall(Callee: StaticFini, Args: {SrcLoc, ThreadNum});
6459
6460 // Add the barrier if requested.
6461 if (NeedsBarrier) {
6462 InsertPointOrErrorTy AfterIP =
6463 createBarrier(Loc: LocationDescription(Builder.saveIP(), DL), Kind: OMPD_for,
6464 /*ForceSimpleCall=*/false, /*CheckCancelFlag=*/false);
6465 if (!AfterIP)
6466 return AfterIP.takeError();
6467 }
6468
6469#ifndef NDEBUG
6470 // Even though we currently do not support applying additional methods to it,
6471 // the chunk loop should remain a canonical loop.
6472 CLI->assertOK();
6473#endif
6474
6475 return DispatchAfter->getFirstInsertionPt();
6476}
6477
6478// Returns an LLVM function to call for executing an OpenMP static worksharing
6479// for loop depending on `type`. Only i32 and i64 are supported by the runtime.
6480// Always interpret integers as unsigned similarly to CanonicalLoopInfo.
6481static FunctionCallee
6482getKmpcForStaticLoopForType(Type *Ty, OpenMPIRBuilder *OMPBuilder,
6483 WorksharingLoopType LoopType) {
6484 unsigned Bitwidth = Ty->getIntegerBitWidth();
6485 Module &M = OMPBuilder->M;
6486 switch (LoopType) {
6487 case WorksharingLoopType::ForStaticLoop:
6488 if (Bitwidth == 32)
6489 return OMPBuilder->getOrCreateRuntimeFunction(
6490 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_for_static_loop_4u);
6491 if (Bitwidth == 64)
6492 return OMPBuilder->getOrCreateRuntimeFunction(
6493 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_for_static_loop_8u);
6494 break;
6495 case WorksharingLoopType::DistributeStaticLoop:
6496 if (Bitwidth == 32)
6497 return OMPBuilder->getOrCreateRuntimeFunction(
6498 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_distribute_static_loop_4u);
6499 if (Bitwidth == 64)
6500 return OMPBuilder->getOrCreateRuntimeFunction(
6501 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_distribute_static_loop_8u);
6502 break;
6503 case WorksharingLoopType::DistributeForStaticLoop:
6504 if (Bitwidth == 32)
6505 return OMPBuilder->getOrCreateRuntimeFunction(
6506 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_distribute_for_static_loop_4u);
6507 if (Bitwidth == 64)
6508 return OMPBuilder->getOrCreateRuntimeFunction(
6509 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_distribute_for_static_loop_8u);
6510 break;
6511 }
6512 if (Bitwidth != 32 && Bitwidth != 64) {
6513 llvm_unreachable("Unknown OpenMP loop iterator bitwidth");
6514 }
6515 llvm_unreachable("Unknown type of OpenMP worksharing loop");
6516}
6517
6518// Inserts a call to proper OpenMP Device RTL function which handles
6519// loop worksharing.
6520static void createTargetLoopWorkshareCall(OpenMPIRBuilder *OMPBuilder,
6521 WorksharingLoopType LoopType,
6522 BasicBlock *InsertBlock, Value *Ident,
6523 Value *LoopBodyArg, Value *TripCount,
6524 Function &LoopBodyFn, bool NoLoop) {
6525 Type *TripCountTy = TripCount->getType();
6526 Module &M = OMPBuilder->M;
6527 IRBuilder<> &Builder = OMPBuilder->Builder;
6528 FunctionCallee RTLFn =
6529 getKmpcForStaticLoopForType(Ty: TripCountTy, OMPBuilder, LoopType);
6530 SmallVector<Value *, 8> RealArgs;
6531 RealArgs.push_back(Elt: Ident);
6532 RealArgs.push_back(Elt: &LoopBodyFn);
6533 RealArgs.push_back(Elt: LoopBodyArg);
6534 RealArgs.push_back(Elt: TripCount);
6535 if (LoopType == WorksharingLoopType::DistributeStaticLoop) {
6536 RealArgs.push_back(Elt: ConstantInt::get(Ty: TripCountTy, V: 0));
6537 RealArgs.push_back(Elt: ConstantInt::get(Ty: Builder.getInt8Ty(), V: 0));
6538 Builder.restoreIP(IP: std::prev(x: InsertBlock->end()));
6539 OMPBuilder->createRuntimeFunctionCall(Callee: RTLFn, Args: RealArgs);
6540 return;
6541 }
6542 FunctionCallee RTLNumThreads = OMPBuilder->getOrCreateRuntimeFunction(
6543 M, FnID: omp::RuntimeFunction::OMPRTL_omp_get_num_threads);
6544 Builder.restoreIP(IP: std::prev(x: InsertBlock->end()));
6545 Value *NumThreads = OMPBuilder->createRuntimeFunctionCall(Callee: RTLNumThreads, Args: {});
6546
6547 RealArgs.push_back(
6548 Elt: Builder.CreateZExtOrTrunc(V: NumThreads, DestTy: TripCountTy, Name: "num.threads.cast"));
6549 RealArgs.push_back(Elt: ConstantInt::get(Ty: TripCountTy, V: 0));
6550 if (LoopType == WorksharingLoopType::DistributeForStaticLoop) {
6551 RealArgs.push_back(Elt: ConstantInt::get(Ty: TripCountTy, V: 0));
6552 RealArgs.push_back(Elt: ConstantInt::get(Ty: Builder.getInt8Ty(), V: NoLoop));
6553 } else {
6554 RealArgs.push_back(Elt: ConstantInt::get(Ty: Builder.getInt8Ty(), V: 0));
6555 }
6556
6557 OMPBuilder->createRuntimeFunctionCall(Callee: RTLFn, Args: RealArgs);
6558}
6559
6560static void workshareLoopTargetCallback(
6561 OpenMPIRBuilder *OMPIRBuilder, CanonicalLoopInfo *CLI, Value *Ident,
6562 Function &OutlinedFn, const SmallVector<Instruction *, 4> &ToBeDeleted,
6563 WorksharingLoopType LoopType, bool NoLoop) {
6564 IRBuilder<> &Builder = OMPIRBuilder->Builder;
6565 BasicBlock *Preheader = CLI->getPreheader();
6566 Value *TripCount = CLI->getTripCount();
6567
6568 // After loop body outling, the loop body contains only set up
6569 // of loop body argument structure and the call to the outlined
6570 // loop body function. Firstly, we need to move setup of loop body args
6571 // into loop preheader.
6572 Preheader->splice(ToIt: std::prev(x: Preheader->end()), FromBB: CLI->getBody(),
6573 FromBeginIt: CLI->getBody()->begin(), FromEndIt: std::prev(x: CLI->getBody()->end()));
6574
6575 // The next step is to remove the whole loop. We do not it need anymore.
6576 // That's why make an unconditional branch from loop preheader to loop
6577 // exit block
6578 Builder.restoreIP(IP: Preheader->end());
6579 Builder.SetCurrentDebugLocation(Preheader->getTerminator()->getDebugLoc());
6580 Preheader->getTerminator()->eraseFromParent();
6581 Builder.CreateBr(Dest: CLI->getExit());
6582
6583 // Delete dead loop blocks
6584 OpenMPIRBuilder::OutlineInfo CleanUpInfo;
6585 SmallPtrSet<BasicBlock *, 32> RegionBlockSet;
6586 SmallVector<BasicBlock *, 32> BlocksToBeRemoved;
6587 CleanUpInfo.EntryBB = CLI->getHeader();
6588 CleanUpInfo.ExitBB = CLI->getExit();
6589 CleanUpInfo.collectBlocks(BlockSet&: RegionBlockSet, BlockVector&: BlocksToBeRemoved);
6590 DeleteDeadBlocks(BBs: BlocksToBeRemoved);
6591
6592 // Find the instruction which corresponds to loop body argument structure
6593 // and remove the call to loop body function instruction.
6594 Value *LoopBodyArg;
6595 User *OutlinedFnUser = OutlinedFn.getUniqueUndroppableUser();
6596 assert(OutlinedFnUser &&
6597 "Expected unique undroppable user of outlined function");
6598 CallInst *OutlinedFnCallInstruction = dyn_cast<CallInst>(Val: OutlinedFnUser);
6599 assert(OutlinedFnCallInstruction && "Expected outlined function call");
6600 assert((OutlinedFnCallInstruction->getParent() == Preheader) &&
6601 "Expected outlined function call to be located in loop preheader");
6602 // Check in case no argument structure has been passed.
6603 if (OutlinedFnCallInstruction->arg_size() > 1)
6604 LoopBodyArg = OutlinedFnCallInstruction->getArgOperand(i: 1);
6605 else
6606 LoopBodyArg = Constant::getNullValue(Ty: Builder.getPtrTy());
6607 OutlinedFnCallInstruction->eraseFromParent();
6608
6609 createTargetLoopWorkshareCall(OMPBuilder: OMPIRBuilder, LoopType, InsertBlock: Preheader, Ident,
6610 LoopBodyArg, TripCount, LoopBodyFn&: OutlinedFn, NoLoop);
6611
6612 for (auto &ToBeDeletedItem : ToBeDeleted)
6613 ToBeDeletedItem->eraseFromParent();
6614 CLI->invalidate();
6615}
6616
6617OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::applyWorkshareLoopTarget(
6618 DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP,
6619 WorksharingLoopType LoopType, bool NeedsBarrier, bool NoLoop,
6620 bool NeedsLastIter) {
6621 uint32_t SrcLocStrSize;
6622 Constant *SrcLocStr = getOrCreateSrcLocStr(DL, SrcLocStrSize);
6623
6624 // Mirrors host runtime reporting of last iteration by in-body computation.
6625 if (NeedsLastIter) {
6626 Type *I32Type = Type::getInt32Ty(C&: M.getContext());
6627 Builder.restoreIP(IP: AllocaIP);
6628 AllocaInst *PLastIter =
6629 Builder.CreateAlloca(Ty: I32Type, ArraySize: nullptr, Name: "p.lastiter");
6630 CLI->setLastIter(PLastIter);
6631
6632 Builder.SetInsertPoint(CLI->getPreheader()->getTerminator());
6633 Builder.CreateStore(Val: ConstantInt::get(Ty: I32Type, V: 0), Ptr: PLastIter);
6634
6635 Builder.SetInsertPoint(CLI->getBody()->getFirstInsertionPt());
6636 Value *TripCount = CLI->getTripCount();
6637 Value *LastIter =
6638 Builder.CreateSub(LHS: TripCount, RHS: ConstantInt::get(Ty: TripCount->getType(), V: 1));
6639 Value *IsLast =
6640 Builder.CreateICmpEQ(LHS: CLI->getIndVar(), RHS: LastIter, Name: "omp.is_last_iter");
6641 Builder.CreateStore(Val: Builder.CreateZExt(V: IsLast, DestTy: I32Type), Ptr: PLastIter);
6642 }
6643
6644 IdentFlag Flag = IdentFlag(0);
6645 switch (LoopType) {
6646 case WorksharingLoopType::ForStaticLoop:
6647 Flag = OMP_IDENT_FLAG_WORK_LOOP;
6648 break;
6649 case WorksharingLoopType::DistributeStaticLoop:
6650 Flag = OMP_IDENT_FLAG_WORK_DISTRIBUTE;
6651 break;
6652 case WorksharingLoopType::DistributeForStaticLoop:
6653 Flag = OMP_IDENT_FLAG_WORK_DISTRIBUTE | OMP_IDENT_FLAG_WORK_LOOP;
6654 break;
6655 }
6656 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize, LocFlags: Flag);
6657
6658 auto OI = std::make_unique<OutlineInfo>();
6659 OI->OuterAllocBB = CLI->getPreheader();
6660 Function *OuterFn = CLI->getPreheader()->getParent();
6661
6662 // Instructions which need to be deleted at the end of code generation
6663 SmallVector<Instruction *, 4> ToBeDeleted;
6664
6665 OI->OuterAllocBB = AllocaIP.getNodeParent();
6666
6667 // Mark the body loop as region which needs to be extracted
6668 OI->EntryBB = CLI->getBody();
6669 OI->ExitBB = CLI->getLatch()->splitBasicBlockBefore(I: CLI->getLatch()->begin(),
6670 BBName: "omp.prelatch");
6671
6672 // Prepare loop body for extraction
6673 Builder.restoreIP(IP: CLI->getPreheader()->begin());
6674
6675 // Insert new loop counter variable which will be used only in loop
6676 // body.
6677 AllocaInst *NewLoopCnt = Builder.CreateAlloca(Ty: CLI->getIndVarType(), ArraySize: 0, Name: "");
6678 Instruction *NewLoopCntLoad =
6679 Builder.CreateLoad(Ty: CLI->getIndVarType(), Ptr: NewLoopCnt);
6680 // New loop counter instructions are redundant in the loop preheader when
6681 // code generation for workshare loop is finshed. That's why mark them as
6682 // ready for deletion.
6683 ToBeDeleted.push_back(Elt: NewLoopCntLoad);
6684 ToBeDeleted.push_back(Elt: NewLoopCnt);
6685
6686 // Analyse loop body region. Find all input variables which are used inside
6687 // loop body region.
6688 SmallPtrSet<BasicBlock *, 32> ParallelRegionBlockSet;
6689 SmallVector<BasicBlock *, 32> Blocks;
6690 OI->collectBlocks(BlockSet&: ParallelRegionBlockSet, BlockVector&: Blocks);
6691
6692 CodeExtractorAnalysisCache CEAC(*OuterFn);
6693 CodeExtractor Extractor(Blocks,
6694 /* DominatorTree */ nullptr,
6695 /* AggregateArgs */ true,
6696 /* BlockFrequencyInfo */ nullptr,
6697 /* BranchProbabilityInfo */ nullptr,
6698 /* AssumptionCache */ nullptr,
6699 /* AllowVarArgs */ true,
6700 /* AllowAlloca */ true,
6701 /* AllocationBlock */ CLI->getPreheader(),
6702 /* DeallocationBlocks */ {},
6703 /* Suffix */ ".omp_wsloop",
6704 /* AggrArgsIn0AddrSpace */ true);
6705
6706 BasicBlock *CommonExit = nullptr;
6707 SetVector<Value *> SinkingCands, HoistingCands;
6708
6709 // Find allocas outside the loop body region which are used inside loop
6710 // body
6711 Extractor.findAllocas(CEAC, SinkCands&: SinkingCands, HoistCands&: HoistingCands, ExitBlock&: CommonExit);
6712
6713 // We need to model loop body region as the function f(cnt, loop_arg).
6714 // That's why we replace loop induction variable by the new counter
6715 // which will be one of loop body function argument
6716 SmallVector<User *> Users(CLI->getIndVar()->user_begin(),
6717 CLI->getIndVar()->user_end());
6718 for (auto Use : Users) {
6719 if (Instruction *Inst = dyn_cast<Instruction>(Val: Use)) {
6720 if (ParallelRegionBlockSet.count(Ptr: Inst->getParent())) {
6721 Inst->replaceUsesOfWith(From: CLI->getIndVar(), To: NewLoopCntLoad);
6722 }
6723 }
6724 }
6725 // Make sure that loop counter variable is not merged into loop body
6726 // function argument structure and it is passed as separate variable
6727 OI->ExcludeArgsFromAggregate.push_back(Elt: NewLoopCntLoad);
6728
6729 // PostOutline CB is invoked when loop body function is outlined and
6730 // loop body is replaced by call to outlined function. We need to add
6731 // call to OpenMP device rtl inside loop preheader. OpenMP device rtl
6732 // function will handle loop control logic.
6733 //
6734 OI->PostOutlineCB = [=, ToBeDeletedVec =
6735 std::move(ToBeDeleted)](Function &OutlinedFn) {
6736 workshareLoopTargetCallback(OMPIRBuilder: this, CLI, Ident, OutlinedFn, ToBeDeleted: ToBeDeletedVec,
6737 LoopType, NoLoop);
6738 };
6739 addOutlineInfo(OI: std::move(OI));
6740
6741 // Keep the barrier outside the outlined loop body so that every thread
6742 // encounters it, including threads that execute no iterations.
6743 if (NeedsBarrier) {
6744 Builder.SetInsertPoint(CLI->getExit()->getTerminator());
6745 // Standalone distribute loops never request a barrier. For both regular
6746 // worksharing loops and combined distribute/for loops, the barrier is
6747 // associated with the worksharing loop, hence OMPD_for.
6748 InsertPointOrErrorTy BarrierIP =
6749 createBarrier(Loc: LocationDescription(Builder.saveIP(), DL), Kind: OMPD_for,
6750 /*ForceSimpleCall=*/false, /*CheckCancelFlag=*/false);
6751 if (!BarrierIP)
6752 return BarrierIP.takeError();
6753 }
6754 return CLI->getAfterIP();
6755}
6756
6757OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::applyWorkshareLoop(
6758 DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP,
6759 bool NeedsBarrier, omp::ScheduleKind SchedKind, Value *ChunkSize,
6760 bool HasSimdModifier, bool HasMonotonicModifier,
6761 bool HasNonmonotonicModifier, bool HasOrderedClause,
6762 WorksharingLoopType LoopType, bool NoLoop, bool HasDistSchedule,
6763 Value *DistScheduleChunkSize, bool NeedsLastIter) {
6764 if (CLI->VirtualCollapse &&
6765 (Config.isTargetDevice() ||
6766 (SchedKind != omp::OMP_SCHEDULE_Static &&
6767 SchedKind != omp::OMP_SCHEDULE_Default) ||
6768 ChunkSize || HasOrderedClause || HasDistSchedule ||
6769 DistScheduleChunkSize || LoopType != WorksharingLoopType::ForStaticLoop))
6770 reportFatalInternalError(
6771 reason: "Virtual collapse requires unchunked static worksharing");
6772 if (Config.isTargetDevice())
6773 return applyWorkshareLoopTarget(DL, CLI, AllocaIP, LoopType, NeedsBarrier,
6774 NoLoop, NeedsLastIter);
6775 OMPScheduleType EffectiveScheduleType = computeOpenMPScheduleType(
6776 ClauseKind: SchedKind, HasChunks: ChunkSize, HasSimdModifier, HasMonotonicModifier,
6777 HasNonmonotonicModifier, HasOrderedClause, HasDistScheduleChunks: DistScheduleChunkSize);
6778
6779 bool IsOrdered = (EffectiveScheduleType & OMPScheduleType::ModifierOrdered) ==
6780 OMPScheduleType::ModifierOrdered;
6781 OMPScheduleType DistScheduleSchedType = OMPScheduleType::None;
6782 if (HasDistSchedule) {
6783 DistScheduleSchedType = DistScheduleChunkSize
6784 ? OMPScheduleType::OrderedDistributeChunked
6785 : OMPScheduleType::OrderedDistribute;
6786 }
6787 switch (EffectiveScheduleType & ~OMPScheduleType::ModifierMask) {
6788 case OMPScheduleType::BaseStatic:
6789 case OMPScheduleType::BaseDistribute:
6790 assert((!ChunkSize || !DistScheduleChunkSize) &&
6791 "No chunk size with static-chunked schedule");
6792 if (IsOrdered && !HasDistSchedule)
6793 return applyDynamicWorkshareLoop(DL, CLI, AllocaIP, SchedType: EffectiveScheduleType,
6794 NeedsBarrier, Chunk: ChunkSize);
6795 // FIXME: Monotonicity ignored?
6796 if (DistScheduleChunkSize)
6797 return applyStaticChunkedWorkshareLoop(
6798 DL, CLI, AllocaIP, NeedsBarrier, ChunkSize, SchedType: EffectiveScheduleType,
6799 DistScheduleChunkSize, DistScheduleSchedType);
6800 return applyStaticWorkshareLoop(DL, CLI, AllocaIP, LoopType, NeedsBarrier,
6801 HasDistSchedule);
6802
6803 case OMPScheduleType::BaseStaticChunked:
6804 case OMPScheduleType::BaseDistributeChunked:
6805 if (IsOrdered && !HasDistSchedule)
6806 return applyDynamicWorkshareLoop(DL, CLI, AllocaIP, SchedType: EffectiveScheduleType,
6807 NeedsBarrier, Chunk: ChunkSize);
6808 // FIXME: Monotonicity ignored?
6809 return applyStaticChunkedWorkshareLoop(
6810 DL, CLI, AllocaIP, NeedsBarrier, ChunkSize, SchedType: EffectiveScheduleType,
6811 DistScheduleChunkSize, DistScheduleSchedType);
6812
6813 case OMPScheduleType::BaseRuntime:
6814 case OMPScheduleType::BaseAuto:
6815 case OMPScheduleType::BaseGreedy:
6816 case OMPScheduleType::BaseBalanced:
6817 case OMPScheduleType::BaseSteal:
6818 case OMPScheduleType::BaseRuntimeSimd:
6819 assert(!ChunkSize &&
6820 "schedule type does not support user-defined chunk sizes");
6821 [[fallthrough]];
6822 case OMPScheduleType::BaseGuidedSimd:
6823 case OMPScheduleType::BaseDynamicChunked:
6824 case OMPScheduleType::BaseGuidedChunked:
6825 case OMPScheduleType::BaseGuidedIterativeChunked:
6826 case OMPScheduleType::BaseGuidedAnalyticalChunked:
6827 case OMPScheduleType::BaseStaticBalancedChunked:
6828 return applyDynamicWorkshareLoop(DL, CLI, AllocaIP, SchedType: EffectiveScheduleType,
6829 NeedsBarrier, Chunk: ChunkSize);
6830
6831 default:
6832 llvm_unreachable("Unknown/unimplemented schedule kind");
6833 }
6834}
6835
6836/// Returns an LLVM function to call for initializing loop bounds using OpenMP
6837/// dynamic scheduling depending on `type`. Only i32 and i64 are supported by
6838/// the runtime. Always interpret integers as unsigned similarly to
6839/// CanonicalLoopInfo.
6840static FunctionCallee
6841getKmpcForDynamicInitForType(Type *Ty, Module &M, OpenMPIRBuilder &OMPBuilder) {
6842 unsigned Bitwidth = Ty->getIntegerBitWidth();
6843 if (Bitwidth == 32)
6844 return OMPBuilder.getOrCreateRuntimeFunction(
6845 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dispatch_init_4u);
6846 if (Bitwidth == 64)
6847 return OMPBuilder.getOrCreateRuntimeFunction(
6848 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dispatch_init_8u);
6849 llvm_unreachable("unknown OpenMP loop iterator bitwidth");
6850}
6851
6852/// Returns an LLVM function to call for updating the next loop using OpenMP
6853/// dynamic scheduling depending on `type`. Only i32 and i64 are supported by
6854/// the runtime. Always interpret integers as unsigned similarly to
6855/// CanonicalLoopInfo.
6856static FunctionCallee
6857getKmpcForDynamicNextForType(Type *Ty, Module &M, OpenMPIRBuilder &OMPBuilder) {
6858 unsigned Bitwidth = Ty->getIntegerBitWidth();
6859 if (Bitwidth == 32)
6860 return OMPBuilder.getOrCreateRuntimeFunction(
6861 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dispatch_next_4u);
6862 if (Bitwidth == 64)
6863 return OMPBuilder.getOrCreateRuntimeFunction(
6864 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dispatch_next_8u);
6865 llvm_unreachable("unknown OpenMP loop iterator bitwidth");
6866}
6867
6868/// Returns an LLVM function to call for finalizing the dynamic loop using
6869/// depending on `type`. Only i32 and i64 are supported by the runtime. Always
6870/// interpret integers as unsigned similarly to CanonicalLoopInfo.
6871static FunctionCallee
6872getKmpcForDynamicFiniForType(Type *Ty, Module &M, OpenMPIRBuilder &OMPBuilder) {
6873 unsigned Bitwidth = Ty->getIntegerBitWidth();
6874 if (Bitwidth == 32)
6875 return OMPBuilder.getOrCreateRuntimeFunction(
6876 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dispatch_fini_4u);
6877 if (Bitwidth == 64)
6878 return OMPBuilder.getOrCreateRuntimeFunction(
6879 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dispatch_fini_8u);
6880 llvm_unreachable("unknown OpenMP loop iterator bitwidth");
6881}
6882
6883OpenMPIRBuilder::InsertPointOrErrorTy
6884OpenMPIRBuilder::applyDynamicWorkshareLoop(DebugLoc DL, CanonicalLoopInfo *CLI,
6885 InsertPointTy AllocaIP,
6886 OMPScheduleType SchedType,
6887 bool NeedsBarrier, Value *Chunk) {
6888 assert(CLI->isValid() && "Requires a valid canonical loop");
6889 assert(!isConflictIP(AllocaIP, CLI->getPreheaderIP()) &&
6890 "Require dedicated allocate IP");
6891 assert(isValidWorkshareLoopScheduleType(SchedType) &&
6892 "Require valid schedule type");
6893
6894 bool Ordered = (SchedType & OMPScheduleType::ModifierOrdered) ==
6895 OMPScheduleType::ModifierOrdered;
6896
6897 // Set up the source location value for OpenMP runtime.
6898 Builder.SetCurrentDebugLocation(DL);
6899
6900 uint32_t SrcLocStrSize;
6901 Constant *SrcLocStr = getOrCreateSrcLocStr(DL, SrcLocStrSize);
6902 Value *SrcLoc =
6903 getOrCreateIdent(SrcLocStr, SrcLocStrSize, LocFlags: OMP_IDENT_FLAG_WORK_LOOP);
6904
6905 // Declare useful OpenMP runtime functions.
6906 Value *IV = CLI->getIndVar();
6907 Type *IVTy = IV->getType();
6908 FunctionCallee DynamicInit = getKmpcForDynamicInitForType(Ty: IVTy, M, OMPBuilder&: *this);
6909 FunctionCallee DynamicNext = getKmpcForDynamicNextForType(Ty: IVTy, M, OMPBuilder&: *this);
6910
6911 // Allocate space for computed loop bounds as expected by the "init" function.
6912 Builder.SetInsertPoint(AllocaIP.getNodeParent()->getFirstNonPHIOrAlloca());
6913 Type *I32Type = Type::getInt32Ty(C&: M.getContext());
6914 Value *PLastIter = Builder.CreateAlloca(Ty: I32Type, ArraySize: nullptr, Name: "p.lastiter");
6915 Value *PLowerBound = Builder.CreateAlloca(Ty: IVTy, ArraySize: nullptr, Name: "p.lowerbound");
6916 Value *PUpperBound = Builder.CreateAlloca(Ty: IVTy, ArraySize: nullptr, Name: "p.upperbound");
6917 Value *PStride = Builder.CreateAlloca(Ty: IVTy, ArraySize: nullptr, Name: "p.stride");
6918 CLI->setLastIter(PLastIter);
6919
6920 // At the end of the preheader, prepare for calling the "init" function by
6921 // storing the current loop bounds into the allocated space. A canonical loop
6922 // always iterates from 0 to trip-count with step 1. Note that "init" expects
6923 // and produces an inclusive upper bound.
6924 BasicBlock *PreHeader = CLI->getPreheader();
6925 Builder.SetInsertPoint(PreHeader->getTerminator());
6926 Constant *One = ConstantInt::get(Ty: IVTy, V: 1);
6927 Builder.CreateStore(Val: One, Ptr: PLowerBound);
6928 Value *UpperBound = CLI->getTripCount();
6929 Builder.CreateStore(Val: UpperBound, Ptr: PUpperBound);
6930 Builder.CreateStore(Val: One, Ptr: PStride);
6931
6932 BasicBlock *Header = CLI->getHeader();
6933 BasicBlock *Exit = CLI->getExit();
6934 BasicBlock *Cond = CLI->getCond();
6935 BasicBlock *Latch = CLI->getLatch();
6936 InsertPointTy AfterIP = CLI->getAfterIP();
6937
6938 // The CLI will be "broken" in the code below, as the loop is no longer
6939 // a valid canonical loop.
6940
6941 if (!Chunk)
6942 Chunk = One;
6943
6944 Value *ThreadNum =
6945 getOrCreateThreadID(Ident: getOrCreateIdent(SrcLocStr, SrcLocStrSize));
6946
6947 Constant *SchedulingType =
6948 ConstantInt::get(Ty: I32Type, V: static_cast<int>(SchedType));
6949
6950 // Call the "init" function.
6951 createRuntimeFunctionCall(Callee: DynamicInit, Args: {SrcLoc, ThreadNum, SchedulingType,
6952 /* LowerBound */ One, UpperBound,
6953 /* step */ One, Chunk});
6954
6955 // An outer loop around the existing one.
6956 BasicBlock *OuterCond = BasicBlock::Create(
6957 Context&: PreHeader->getContext(), Name: Twine(PreHeader->getName()) + ".outer.cond",
6958 Parent: PreHeader->getParent());
6959 // This needs to be 32-bit always, so can't use the IVTy Zero above.
6960 Builder.SetInsertPoint(OuterCond->getFirstInsertionPt());
6961 Value *Res = createRuntimeFunctionCall(
6962 Callee: DynamicNext,
6963 Args: {SrcLoc, ThreadNum, PLastIter, PLowerBound, PUpperBound, PStride});
6964 Constant *Zero32 = ConstantInt::get(Ty: I32Type, V: 0);
6965 Value *MoreWork = Builder.CreateCmp(Pred: CmpInst::ICMP_NE, LHS: Res, RHS: Zero32);
6966 Value *LowerBound =
6967 Builder.CreateSub(LHS: Builder.CreateLoad(Ty: IVTy, Ptr: PLowerBound), RHS: One, Name: "lb");
6968 Builder.CreateCondBr(Cond: MoreWork, True: Header, False: Exit);
6969
6970 // Change PHI-node in loop header to use outer cond rather than preheader,
6971 // and set IV to the LowerBound.
6972 Instruction *Phi = &Header->front();
6973 auto *PI = cast<PHINode>(Val: Phi);
6974 PI->setIncomingBlock(i: 0, BB: OuterCond);
6975 PI->setIncomingValue(i: 0, V: LowerBound);
6976
6977 // Then set the pre-header to jump to the OuterCond
6978 Instruction *Term = PreHeader->getTerminator();
6979 auto *Br = cast<UncondBrInst>(Val: Term);
6980 Br->setSuccessor(OuterCond);
6981
6982 // Modify the inner condition:
6983 // * Use the UpperBound returned from the DynamicNext call.
6984 // * jump to the loop outer loop when done with one of the inner loops.
6985 Builder.SetInsertPoint(Cond->getFirstInsertionPt());
6986 UpperBound = Builder.CreateLoad(Ty: IVTy, Ptr: PUpperBound, Name: "ub");
6987 Instruction *Comp = &*Builder.GetInsertPoint();
6988 auto *CI = cast<CmpInst>(Val: Comp);
6989 CI->setOperand(i_nocapture: 1, Val_nocapture: UpperBound);
6990 // Redirect the inner exit to branch to outer condition.
6991 Instruction *Branch = &Cond->back();
6992 auto *BI = cast<CondBrInst>(Val: Branch);
6993 assert(BI->getSuccessor(1) == Exit);
6994 BI->setSuccessor(idx: 1, NewSucc: OuterCond);
6995
6996 // Call the "fini" function if "ordered" is present in wsloop directive.
6997 if (Ordered) {
6998 Builder.SetInsertPoint(&Latch->back());
6999 FunctionCallee DynamicFini = getKmpcForDynamicFiniForType(Ty: IVTy, M, OMPBuilder&: *this);
7000 createRuntimeFunctionCall(Callee: DynamicFini, Args: {SrcLoc, ThreadNum});
7001 }
7002
7003 // Add the barrier if requested.
7004 if (NeedsBarrier) {
7005 Builder.SetInsertPoint(&Exit->back());
7006 InsertPointOrErrorTy BarrierIP =
7007 createBarrier(Loc: LocationDescription(Builder.saveIP(), DL),
7008 Kind: omp::Directive::OMPD_for, /* ForceSimpleCall */ false,
7009 /* CheckCancelFlag */ false);
7010 if (!BarrierIP)
7011 return BarrierIP.takeError();
7012 }
7013
7014 CLI->invalidate();
7015 return AfterIP;
7016}
7017
7018/// Redirect all edges that branch to \p OldTarget to \p NewTarget. That is,
7019/// after this \p OldTarget will be orphaned.
7020static void redirectAllPredecessorsTo(BasicBlock *OldTarget,
7021 BasicBlock *NewTarget, DebugLoc DL) {
7022 for (BasicBlock *Pred : make_early_inc_range(Range: predecessors(BB: OldTarget)))
7023 redirectTo(Source: Pred, Target: NewTarget, DL);
7024}
7025
7026static void removeUnusedBlocksFromParent(ArrayRef<BasicBlock *> BBs) {
7027 SmallPtrSet<BasicBlock *, 8> InternalBBs(from_range, BBs);
7028 // We add a block to BBsToKeep iff we have proven it has an external use.
7029 SmallPtrSet<BasicBlock *, 8> BBsToKeep;
7030
7031 while (true) {
7032 bool Changed = false;
7033
7034 for (BasicBlock *BB : BBs) {
7035 if (BBsToKeep.contains(Ptr: BB))
7036 continue;
7037
7038 for (Use &U : BB->uses()) {
7039 auto *UseInst = dyn_cast<Instruction>(Val: U.getUser());
7040 if (!UseInst)
7041 continue;
7042 BasicBlock *UseBB = UseInst->getParent();
7043 if (!InternalBBs.contains(Ptr: UseBB) || BBsToKeep.contains(Ptr: UseBB)) {
7044 BBsToKeep.insert(Ptr: BB);
7045 Changed = true;
7046 break;
7047 }
7048 }
7049 }
7050
7051 if (!Changed)
7052 break;
7053 }
7054
7055 SmallVector<BasicBlock *> BBsToDelete = filter_to_vector(
7056 C&: BBs, Pred: [&BBsToKeep](BasicBlock *BB) { return !BBsToKeep.contains(Ptr: BB); });
7057 DeleteDeadBlocks(BBs: BBsToDelete);
7058}
7059
7060/// Physical handles captured before the source descriptors are invalidated.
7061struct OpenMPIRBuilder::VirtualRowPlan {
7062 SmallVector<PHINode *, 4> IndVars;
7063 SmallVector<BasicBlock *, 4> Preheaders;
7064 SmallVector<BasicBlock *, 4> Latches;
7065 CmpInst *InnerCmp;
7066};
7067
7068OpenMPIRBuilder::VirtualRowPlan
7069OpenMPIRBuilder::planVirtualRowLoop(ArrayRef<CanonicalLoopInfo *> Loops) {
7070 // The executor inserts a guard before the outer latch. An indirect jump
7071 // by block address could bypass that guard.
7072 if (isa<IndirectBrInst>(Val: Loops.back()->getAfter()->getTerminator()))
7073 reportFatalInternalError(
7074 reason: "Virtual collapse requires unobserved canonical loop controls");
7075 // Save the original blocks and IVs before the executor changes the CFG.
7076 VirtualRowPlan Plan;
7077 Plan.Latches.push_back(Elt: Loops.front()->getLatch());
7078 for (CanonicalLoopInfo *Loop : Loops) {
7079 Plan.IndVars.push_back(Elt: cast<PHINode>(Val: Loop->getIndVar()));
7080 Plan.Preheaders.push_back(Elt: Loop->getPreheader());
7081 }
7082 // The inner loop must return directly to the latch where the guard is added.
7083 if (Loops.front()->getBody()->getUniqueSuccessor() !=
7084 Plan.Preheaders.back() ||
7085 Loops.back()->getAfter()->getSingleSuccessor() != Plan.Latches.front())
7086 reportFatalInternalError(reason: "Virtual collapse requires direct nesting");
7087 Plan.InnerCmp = cast<CmpInst>(Val: &Loops.back()->getCond()->front());
7088 return Plan;
7089}
7090
7091void OpenMPIRBuilder::createVirtualRowExecutor(CanonicalLoopInfo *Result,
7092 const VirtualRowPlan &Plan,
7093 ArrayRef<Value *> TripCounts,
7094 Value *Begin, Value *Count,
7095 BasicBlock *Entry) {
7096 DebugLoc DL = Builder.getCurrentDebugLocation();
7097 BasicBlock *OuterHeader = Plan.IndVars.front()->getParent();
7098 Type *IVType = Begin->getType();
7099 Constant *Zero = ConstantInt::get(Ty: IVType, V: 0);
7100 // A row is one invocation of the inner loop. Decode the first logical
7101 // iteration once, before entering the original loop nest.
7102 Builder.SetInsertPoint(Entry);
7103 Value *InnerBegin =
7104 Builder.CreateURem(LHS: Begin, RHS: TripCounts.back(), Name: "omp_collapsed.begin.1");
7105 Value *OuterBegin =
7106 Builder.CreateUDiv(LHS: Begin, RHS: TripCounts.back(), Name: "omp_collapsed.begin.0");
7107 // Include the skipped prefix of the first row so each completed row
7108 // removes one full inner trip count from Remaining.
7109 Value *InitialRemaining =
7110 Builder.CreateAdd(LHS: Count, RHS: InnerBegin, Name: "omp_collapsed.row.remaining");
7111 Builder.CreateBr(Dest: OuterHeader);
7112
7113 Builder.SetInsertPoint(OuterHeader->getFirstNonPHIIt());
7114 Builder.SetCurrentDebugLocation(DL);
7115 PHINode *Remaining =
7116 Builder.CreatePHI(Ty: IVType, NumReservedValues: 2, Name: "omp_collapsed.row.remaining.0");
7117 Remaining->addIncoming(V: InitialRemaining, BB: Entry);
7118 // Only the first row starts at InnerBegin; subsequent rows start at zero.
7119 // For an inner trip count of one, the existing zero start is sufficient.
7120 if (!cast<ConstantInt>(Val: TripCounts.back())->isOne()) {
7121 PHINode *Start =
7122 Builder.CreatePHI(Ty: IVType, NumReservedValues: 2, Name: "omp_collapsed.row.start.1.0");
7123 Start->addIncoming(V: InnerBegin, BB: Entry);
7124 Start->addIncoming(V: Zero, BB: Plan.Latches.front());
7125 Plan.IndVars.back()->setIncomingValue(i: 0, V: Start);
7126 }
7127 // Saved inner-preheader insertions must not change the IV PHI's entry edge.
7128 // Keep the original branch as their anchor and give it a PHI-free successor.
7129 auto *InnerBr = cast<UncondBrInst>(Val: Plan.Preheaders.back()->getTerminator());
7130 BasicBlock *InnerHeader = Plan.IndVars.back()->getParent();
7131 BasicBlock *InnerEntry =
7132 BasicBlock::Create(Context&: M.getContext(), Name: "omp_collapsed.row.preheader",
7133 Parent: InnerHeader->getParent(), InsertBefore: InnerHeader);
7134 Builder.SetInsertPoint(InnerEntry);
7135 Builder.SetCurrentDebugLocation(DL);
7136 Builder.CreateBr(Dest: InnerHeader);
7137 InnerBr->setSuccessor(InnerEntry);
7138 Plan.IndVars.back()->setIncomingBlock(i: 0, BB: InnerEntry);
7139 // The final row ends at Remaining; earlier rows keep their original bound.
7140 Builder.SetInsertPoint(Plan.Preheaders.back()->getTerminator());
7141 Builder.SetCurrentDebugLocation(DL);
7142 Value *IsFinalRow = Builder.CreateICmpULE(LHS: Remaining, RHS: TripCounts.back(),
7143 Name: "omp_collapsed.row.is_final");
7144 Value *InnerStop = Builder.CreateSelect(
7145 C: IsFinalRow, True: Remaining, False: TripCounts.back(), Name: "omp_collapsed.row.stop");
7146 Plan.InnerCmp->setOperand(i_nocapture: 1, Val_nocapture: InnerStop);
7147
7148 // Only non-final rows reach the outer latch, so Remaining is greater than
7149 // the inner trip count and this subtraction cannot underflow.
7150 BasicBlock *Latch = Plan.Latches.front();
7151 Builder.SetInsertPoint(Latch->getTerminator());
7152 Builder.SetCurrentDebugLocation(DL);
7153 Value *RemainingNext = Builder.CreateSub(LHS: Remaining, RHS: TripCounts.back(),
7154 Name: "omp_collapsed.row.remaining.next",
7155 /*HasNUW=*/true);
7156 Remaining->addIncoming(V: RemainingNext, BB: Latch);
7157 // Check completion after the inner loop and its continuation, before
7158 // incrementing the outer IV. The final row exits the whole nest.
7159 BasicBlock *Guard =
7160 Latch->splitBasicBlockBefore(I: Latch->begin(), BBName: "omp_collapsed.row.done");
7161 Guard->getTerminator()->eraseFromParent();
7162 Builder.SetInsertPoint(Guard);
7163 Builder.CreateCondBr(Cond: IsFinalRow, True: Result->getExit(), False: Latch);
7164 Plan.IndVars.front()->setIncomingValue(i: 0, V: OuterBegin);
7165 Plan.IndVars.front()->setIncomingBlock(i: 0, BB: Entry);
7166}
7167
7168CanonicalLoopInfo *OpenMPIRBuilder::createVirtualCollapsedLoop(
7169 DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
7170 const VirtualRowPlan &Plan, ArrayRef<Value *> TripCounts,
7171 Value *CollapsedTripCount) {
7172 // Reuse the outer loop's control blocks; the physical nest stays in place.
7173 LoopInfos.emplace_front();
7174 CanonicalLoopInfo *Result = &LoopInfos.front();
7175 Result->Header = Loops.front()->getHeader();
7176 Result->Cond = Loops.front()->getCond();
7177 Result->Latch = Loops.front()->getLatch();
7178 Result->Exit = Loops.front()->getExit();
7179 BasicBlock *Preheader = Plan.Preheaders.front();
7180 Function *F = Result->getFunction();
7181 Type *IVType = CollapsedTripCount->getType();
7182 Constant *Zero = ConstantInt::get(Ty: IVType, V: 0);
7183 // Keep initialization separate from decoding so an empty range skips both
7184 // division and the loop nest.
7185 BasicBlock *Init = BasicBlock::Create(Context&: M.getContext(), Name: "omp_collapsed.init", Parent: F,
7186 InsertBefore: Result->getHeader());
7187 BasicBlock *Entry = BasicBlock::Create(Context&: M.getContext(), Name: "omp_collapsed.decode",
7188 Parent: F, InsertBefore: Result->getHeader());
7189 Builder.SetCurrentDebugLocation(DL);
7190 Builder.SetInsertPoint(Init);
7191 // Keep explicit range copies until scheduling replaces operand zero. Create
7192 // the instructions directly so the initial constants are not folded away.
7193 auto *Begin = Builder.Insert(I: BinaryOperator::CreateAdd(V1: Zero, V2: Zero),
7194 Name: "omp_collapsed.range.begin");
7195 auto *Count =
7196 Builder.Insert(I: BinaryOperator::CreateAdd(V1: CollapsedTripCount, V2: Zero),
7197 Name: "omp_collapsed.range.count");
7198 Builder.CreateCondBr(Cond: Builder.CreateICmpNE(LHS: Count, RHS: Zero), True: Entry,
7199 False: Result->getExit());
7200 Result->VirtualCollapse =
7201 CanonicalLoopInfo::VirtualCollapseInfo{.Begin: Begin, .Count: Count};
7202 // Route the preheader through Init. The executor reconnects the outer IV
7203 // PHI to the decode block.
7204 cast<UncondBrInst>(Val: Preheader->getTerminator())->setSuccessor(Init);
7205 createVirtualRowExecutor(Result, Plan, TripCounts, Begin, Count, Entry);
7206 // The transformed nest is now represented by Result, not the input loops.
7207 for (CanonicalLoopInfo *Loop : Loops)
7208 Loop->invalidate();
7209#ifndef NDEBUG
7210 Result->assertOK();
7211#endif
7212 return Result;
7213}
7214
7215void OpenMPIRBuilder::bindCollapsedRange(CanonicalLoopInfo *Loop, Value *Begin,
7216 Value *Count) {
7217 // Update only the copied values; the range has no PHI predecessor to rebind.
7218 Loop->VirtualCollapse->Begin->setOperand(i_nocapture: 0, Val_nocapture: Begin);
7219 Loop->VirtualCollapse->Count->setOperand(i_nocapture: 0, Val_nocapture: Count);
7220}
7221
7222CanonicalLoopInfo *
7223OpenMPIRBuilder::collapseLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
7224 InsertPointTy ComputeIP) {
7225 assert(Loops.size() >= 1 && "At least one loop required");
7226 size_t NumLoops = Loops.size();
7227
7228 // Nothing to do if there is already just one loop.
7229 if (NumLoops == 1)
7230 return Loops.front();
7231
7232 if (Config.CollapseMapping == CollapseIVMapping::Virtual) {
7233 if (Loops.size() != 2)
7234 reportFatalInternalError(
7235 reason: "Virtual collapse requires two constant-count loops");
7236 SmallVector<Value *, 4> TripCounts;
7237 for (CanonicalLoopInfo *Loop : Loops) {
7238 if (Loop->VirtualCollapse)
7239 Loop->reportUnsupportedVirtualCollapse();
7240 Value *TC = Loop->getTripCount();
7241 if (!isa<ConstantInt>(Val: TC))
7242 reportFatalInternalError(
7243 reason: "Virtual collapse requires two constant-count loops");
7244 TripCounts.push_back(Elt: TC);
7245 }
7246 VirtualRowPlan Plan = planVirtualRowLoop(Loops);
7247 // Constant counts fold the product without emitting instructions.
7248 Value *CollapsedTripCount =
7249 Builder.CreateNUWMul(LHS: TripCounts.front(), RHS: TripCounts.back());
7250 return createVirtualCollapsedLoop(DL, Loops, Plan, TripCounts,
7251 CollapsedTripCount);
7252 }
7253
7254 CanonicalLoopInfo *Outermost = Loops.front();
7255 CanonicalLoopInfo *Innermost = Loops.back();
7256 BasicBlock *OrigPreheader = Outermost->getPreheader();
7257 BasicBlock *OrigAfter = Outermost->getAfter();
7258 Function *F = OrigPreheader->getParent();
7259
7260 // Loop control blocks that may become orphaned later.
7261 SmallVector<BasicBlock *, 12> OldControlBBs;
7262 OldControlBBs.reserve(N: 6 * Loops.size());
7263 for (CanonicalLoopInfo *Loop : Loops)
7264 Loop->collectControlBlocks(BBs&: OldControlBBs);
7265
7266 // Setup the IRBuilder for inserting the trip count computation.
7267 Builder.SetCurrentDebugLocation(DL);
7268 if (ComputeIP.isValid())
7269 Builder.restoreIP(IP: ComputeIP);
7270 else
7271 Builder.restoreIP(IP: Outermost->getPreheaderIP());
7272
7273 // Derive the collapsed' loop trip count.
7274 // TODO: Find common/largest indvar type.
7275 Value *CollapsedTripCount = nullptr;
7276 for (CanonicalLoopInfo *L : Loops) {
7277 assert(L->isValid() &&
7278 "All loops to collapse must be valid canonical loops");
7279 Value *OrigTripCount = L->getTripCount();
7280 if (!CollapsedTripCount) {
7281 CollapsedTripCount = OrigTripCount;
7282 continue;
7283 }
7284
7285 // TODO: Enable UndefinedSanitizer to diagnose an overflow here.
7286 CollapsedTripCount =
7287 Builder.CreateNUWMul(LHS: CollapsedTripCount, RHS: OrigTripCount);
7288 }
7289
7290 // Create the collapsed loop control flow.
7291 CanonicalLoopInfo *Result =
7292 createLoopSkeleton(DL, TripCount: CollapsedTripCount, F,
7293 PreInsertBefore: OrigPreheader->getNextNode(), PostInsertBefore: OrigAfter, Name: "collapsed",
7294 /*IsCollapsed=*/true);
7295
7296 // Build the collapsed loop body code.
7297 // Start with deriving the input loop induction variables from the collapsed
7298 // one, using a divmod scheme. To preserve the original loops' order, the
7299 // innermost loop use the least significant bits.
7300 Builder.restoreIP(IP: Result->getBodyIP());
7301
7302 Value *Leftover = Result->getIndVar();
7303 SmallVector<Value *> NewIndVars;
7304 NewIndVars.resize(N: NumLoops);
7305 for (int i = NumLoops - 1; i >= 1; --i) {
7306 Value *OrigTripCount = Loops[i]->getTripCount();
7307
7308 Value *NewIndVar = Builder.CreateURem(LHS: Leftover, RHS: OrigTripCount);
7309 NewIndVars[i] = NewIndVar;
7310
7311 Leftover = Builder.CreateUDiv(LHS: Leftover, RHS: OrigTripCount);
7312 }
7313 // Outermost loop gets all the remaining bits.
7314 NewIndVars[0] = Leftover;
7315
7316 // Construct the loop body control flow.
7317 // We progressively construct the branch structure following in direction of
7318 // the control flow, from the leading in-between code, the loop nest body, the
7319 // trailing in-between code, and rejoining the collapsed loop's latch.
7320 // ContinueBlock and ContinuePred keep track of the source(s) of next edge. If
7321 // the ContinueBlock is set, continue with that block. If ContinuePred, use
7322 // its predecessors as sources.
7323 BasicBlock *ContinueBlock = Result->getBody();
7324 BasicBlock *ContinuePred = nullptr;
7325 auto ContinueWith = [&ContinueBlock, &ContinuePred, DL](BasicBlock *Dest,
7326 BasicBlock *NextSrc) {
7327 if (ContinueBlock)
7328 redirectTo(Source: ContinueBlock, Target: Dest, DL);
7329 else
7330 redirectAllPredecessorsTo(OldTarget: ContinuePred, NewTarget: Dest, DL);
7331
7332 ContinueBlock = nullptr;
7333 ContinuePred = NextSrc;
7334 };
7335
7336 // The code before the nested loop of each level.
7337 // Because we are sinking it into the nest, it will be executed more often
7338 // that the original loop. More sophisticated schemes could keep track of what
7339 // the in-between code is and instantiate it only once per thread.
7340 for (size_t i = 0; i < NumLoops - 1; ++i)
7341 ContinueWith(Loops[i]->getBody(), Loops[i + 1]->getHeader());
7342
7343 // Connect the loop nest body.
7344 ContinueWith(Innermost->getBody(), Innermost->getLatch());
7345
7346 // The code after the nested loop at each level.
7347 for (size_t i = NumLoops - 1; i > 0; --i)
7348 ContinueWith(Loops[i]->getAfter(), Loops[i - 1]->getLatch());
7349
7350 // Connect the finished loop to the collapsed loop latch.
7351 ContinueWith(Result->getLatch(), nullptr);
7352
7353 // Replace the input loops with the new collapsed loop.
7354 redirectTo(Source: Outermost->getPreheader(), Target: Result->getPreheader(), DL);
7355 redirectTo(Source: Result->getAfter(), Target: Outermost->getAfter(), DL);
7356
7357 // Replace the input loop indvars with the derived ones.
7358 for (size_t i = 0; i < NumLoops; ++i)
7359 Loops[i]->getIndVar()->replaceAllUsesWith(V: NewIndVars[i]);
7360
7361 // Remove unused parts of the input loops.
7362 removeUnusedBlocksFromParent(BBs: OldControlBBs);
7363
7364 for (CanonicalLoopInfo *L : Loops)
7365 L->invalidate();
7366
7367#ifndef NDEBUG
7368 Result->assertOK();
7369#endif
7370 return Result;
7371}
7372
7373std::vector<CanonicalLoopInfo *>
7374OpenMPIRBuilder::tileLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
7375 ArrayRef<Value *> TileSizes) {
7376 assert(TileSizes.size() == Loops.size() &&
7377 "Must pass as many tile sizes as there are loops");
7378 int NumLoops = Loops.size();
7379 assert(NumLoops >= 1 && "At least one loop to tile required");
7380
7381 CanonicalLoopInfo *OutermostLoop = Loops.front();
7382 CanonicalLoopInfo *InnermostLoop = Loops.back();
7383 Function *F = OutermostLoop->getBody()->getParent();
7384 BasicBlock *InnerEnter = InnermostLoop->getBody();
7385 BasicBlock *InnerLatch = InnermostLoop->getLatch();
7386
7387 // Loop control blocks that may become orphaned later.
7388 SmallVector<BasicBlock *, 12> OldControlBBs;
7389 OldControlBBs.reserve(N: 6 * Loops.size());
7390 for (CanonicalLoopInfo *Loop : Loops)
7391 Loop->collectControlBlocks(BBs&: OldControlBBs);
7392
7393 // Collect original trip counts and induction variable to be accessible by
7394 // index. Also, the structure of the original loops is not preserved during
7395 // the construction of the tiled loops, so do it before we scavenge the BBs of
7396 // any original CanonicalLoopInfo.
7397 SmallVector<Value *, 4> OrigTripCounts, OrigIndVars;
7398 for (CanonicalLoopInfo *L : Loops) {
7399 assert(L->isValid() && "All input loops must be valid canonical loops");
7400 OrigTripCounts.push_back(Elt: L->getTripCount());
7401 OrigIndVars.push_back(Elt: L->getIndVar());
7402 }
7403
7404 // Collect the code between loop headers. These may contain SSA definitions
7405 // that are used in the loop nest body. To be usable with in the innermost
7406 // body, these BasicBlocks will be sunk into the loop nest body. That is,
7407 // these instructions may be executed more often than before the tiling.
7408 // TODO: It would be sufficient to only sink them into body of the
7409 // corresponding tile loop.
7410 SmallVector<std::pair<BasicBlock *, BasicBlock *>, 4> InbetweenCode;
7411 for (int i = 0; i < NumLoops - 1; ++i) {
7412 CanonicalLoopInfo *Surrounding = Loops[i];
7413 CanonicalLoopInfo *Nested = Loops[i + 1];
7414
7415 BasicBlock *EnterBB = Surrounding->getBody();
7416 BasicBlock *ExitBB = Nested->getHeader();
7417 InbetweenCode.emplace_back(Args&: EnterBB, Args&: ExitBB);
7418 }
7419
7420 // Compute the trip counts of the floor loops.
7421 Builder.SetCurrentDebugLocation(DL);
7422 Builder.restoreIP(IP: OutermostLoop->getPreheaderIP());
7423 SmallVector<Value *, 4> FloorCompleteCount, FloorCount, FloorRems;
7424 for (int i = 0; i < NumLoops; ++i) {
7425 Value *TileSize = TileSizes[i];
7426 Value *OrigTripCount = OrigTripCounts[i];
7427 Type *IVType = OrigTripCount->getType();
7428
7429 Value *FloorCompleteTripCount = Builder.CreateUDiv(LHS: OrigTripCount, RHS: TileSize);
7430 Value *FloorTripRem = Builder.CreateURem(LHS: OrigTripCount, RHS: TileSize);
7431
7432 // 0 if tripcount divides the tilesize, 1 otherwise.
7433 // 1 means we need an additional iteration for a partial tile.
7434 //
7435 // Unfortunately we cannot just use the roundup-formula
7436 // (tripcount + tilesize - 1)/tilesize
7437 // because the summation might overflow. We do not want introduce undefined
7438 // behavior when the untiled loop nest did not.
7439 Value *FloorTripOverflow =
7440 Builder.CreateICmpNE(LHS: FloorTripRem, RHS: ConstantInt::get(Ty: IVType, V: 0));
7441
7442 FloorTripOverflow = Builder.CreateZExt(V: FloorTripOverflow, DestTy: IVType);
7443 Value *FloorTripCount =
7444 Builder.CreateAdd(LHS: FloorCompleteTripCount, RHS: FloorTripOverflow,
7445 Name: "omp_floor" + Twine(i) + ".tripcount", HasNUW: true);
7446
7447 // Remember some values for later use.
7448 FloorCompleteCount.push_back(Elt: FloorCompleteTripCount);
7449 FloorCount.push_back(Elt: FloorTripCount);
7450 FloorRems.push_back(Elt: FloorTripRem);
7451 }
7452
7453 // Generate the new loop nest, from the outermost to the innermost.
7454 std::vector<CanonicalLoopInfo *> Result;
7455 Result.reserve(n: NumLoops * 2);
7456
7457 // The basic block of the surrounding loop that enters the nest generated
7458 // loop.
7459 BasicBlock *Enter = OutermostLoop->getPreheader();
7460
7461 // The basic block of the surrounding loop where the inner code should
7462 // continue.
7463 BasicBlock *Continue = OutermostLoop->getAfter();
7464
7465 // Where the next loop basic block should be inserted.
7466 BasicBlock *OutroInsertBefore = InnermostLoop->getExit();
7467
7468 auto EmbeddNewLoop =
7469 [this, DL, F, InnerEnter, &Enter, &Continue, &OutroInsertBefore](
7470 Value *TripCount, const Twine &Name) -> CanonicalLoopInfo * {
7471 CanonicalLoopInfo *EmbeddedLoop = createLoopSkeleton(
7472 DL, TripCount, F, PreInsertBefore: InnerEnter, PostInsertBefore: OutroInsertBefore, Name);
7473 redirectTo(Source: Enter, Target: EmbeddedLoop->getPreheader(), DL);
7474 redirectTo(Source: EmbeddedLoop->getAfter(), Target: Continue, DL);
7475
7476 // Setup the position where the next embedded loop connects to this loop.
7477 Enter = EmbeddedLoop->getBody();
7478 Continue = EmbeddedLoop->getLatch();
7479 OutroInsertBefore = EmbeddedLoop->getLatch();
7480 return EmbeddedLoop;
7481 };
7482
7483 auto EmbeddNewLoops = [&Result, &EmbeddNewLoop](ArrayRef<Value *> TripCounts,
7484 const Twine &NameBase) {
7485 for (auto P : enumerate(First&: TripCounts)) {
7486 CanonicalLoopInfo *EmbeddedLoop =
7487 EmbeddNewLoop(P.value(), NameBase + Twine(P.index()));
7488 Result.push_back(x: EmbeddedLoop);
7489 }
7490 };
7491
7492 EmbeddNewLoops(FloorCount, "floor");
7493
7494 // Within the innermost floor loop, emit the code that computes the tile
7495 // sizes.
7496 Builder.SetInsertPoint(Enter->getTerminator());
7497 SmallVector<Value *, 4> TileCounts;
7498 for (int i = 0; i < NumLoops; ++i) {
7499 CanonicalLoopInfo *FloorLoop = Result[i];
7500 Value *TileSize = TileSizes[i];
7501
7502 Value *FloorIsEpilogue =
7503 Builder.CreateICmpEQ(LHS: FloorLoop->getIndVar(), RHS: FloorCompleteCount[i]);
7504 Value *TileTripCount =
7505 Builder.CreateSelect(C: FloorIsEpilogue, True: FloorRems[i], False: TileSize);
7506
7507 TileCounts.push_back(Elt: TileTripCount);
7508 }
7509
7510 // Create the tile loops.
7511 EmbeddNewLoops(TileCounts, "tile");
7512
7513 // Insert the inbetween code into the body.
7514 BasicBlock *BodyEnter = Enter;
7515 BasicBlock *BodyEntered = nullptr;
7516 for (std::pair<BasicBlock *, BasicBlock *> P : InbetweenCode) {
7517 BasicBlock *EnterBB = P.first;
7518 BasicBlock *ExitBB = P.second;
7519
7520 if (BodyEnter)
7521 redirectTo(Source: BodyEnter, Target: EnterBB, DL);
7522 else
7523 redirectAllPredecessorsTo(OldTarget: BodyEntered, NewTarget: EnterBB, DL);
7524
7525 BodyEnter = nullptr;
7526 BodyEntered = ExitBB;
7527 }
7528
7529 // Append the original loop nest body into the generated loop nest body.
7530 if (BodyEnter)
7531 redirectTo(Source: BodyEnter, Target: InnerEnter, DL);
7532 else
7533 redirectAllPredecessorsTo(OldTarget: BodyEntered, NewTarget: InnerEnter, DL);
7534 redirectAllPredecessorsTo(OldTarget: InnerLatch, NewTarget: Continue, DL);
7535
7536 // Replace the original induction variable with an induction variable computed
7537 // from the tile and floor induction variables.
7538 Builder.restoreIP(IP: Result.back()->getBodyIP());
7539 for (int i = 0; i < NumLoops; ++i) {
7540 CanonicalLoopInfo *FloorLoop = Result[i];
7541 CanonicalLoopInfo *TileLoop = Result[NumLoops + i];
7542 Value *OrigIndVar = OrigIndVars[i];
7543 Value *Size = TileSizes[i];
7544
7545 Value *Scale =
7546 Builder.CreateMul(LHS: Size, RHS: FloorLoop->getIndVar(), Name: {}, /*HasNUW=*/true);
7547 Value *Shift =
7548 Builder.CreateAdd(LHS: Scale, RHS: TileLoop->getIndVar(), Name: {}, /*HasNUW=*/true);
7549 OrigIndVar->replaceAllUsesWith(V: Shift);
7550 }
7551
7552 // Remove unused parts of the original loops.
7553 removeUnusedBlocksFromParent(BBs: OldControlBBs);
7554
7555 for (CanonicalLoopInfo *L : Loops)
7556 L->invalidate();
7557
7558#ifndef NDEBUG
7559 for (CanonicalLoopInfo *GenL : Result)
7560 GenL->assertOK();
7561#endif
7562 return Result;
7563}
7564
7565/// Attach metadata \p Properties to the basic block described by \p BB. If the
7566/// basic block already has metadata, the basic block properties are appended.
7567static void addBasicBlockMetadata(BasicBlock *BB,
7568 ArrayRef<Metadata *> Properties) {
7569 // Nothing to do if no property to attach.
7570 if (Properties.empty())
7571 return;
7572
7573 LLVMContext &Ctx = BB->getContext();
7574 SmallVector<Metadata *> NewProperties;
7575 NewProperties.push_back(Elt: nullptr);
7576
7577 // If the basic block already has metadata, prepend it to the new metadata.
7578 MDNode *Existing = BB->getTerminator()->getMetadata(KindID: LLVMContext::MD_loop);
7579 if (Existing)
7580 append_range(C&: NewProperties, R: drop_begin(RangeOrContainer: Existing->operands(), N: 1));
7581
7582 append_range(C&: NewProperties, R&: Properties);
7583 MDNode *BasicBlockID = MDNode::getDistinct(Context&: Ctx, MDs: NewProperties);
7584 BasicBlockID->replaceOperandWith(I: 0, New: BasicBlockID);
7585
7586 BB->getTerminator()->setMetadata(KindID: LLVMContext::MD_loop, Node: BasicBlockID);
7587}
7588
7589/// Attach loop metadata \p Properties to the loop described by \p Loop. If the
7590/// loop already has metadata, the loop properties are appended.
7591static void addLoopMetadata(CanonicalLoopInfo *Loop,
7592 ArrayRef<Metadata *> Properties) {
7593 assert(Loop->isValid() && "Expecting a valid CanonicalLoopInfo");
7594
7595 // Attach metadata to the loop's latch
7596 BasicBlock *Latch = Loop->getLatch();
7597 assert(Latch && "A valid CanonicalLoopInfo must have a unique latch");
7598 addBasicBlockMetadata(BB: Latch, Properties);
7599}
7600
7601/// Attach llvm.access.group metadata to the memref instructions of \p Block
7602static void addAccessGroupMetadata(BasicBlock *Block, MDNode *AccessGroup,
7603 LoopInfo &LI) {
7604 for (Instruction &I : *Block) {
7605 if (I.mayReadOrWriteMemory()) {
7606 // TODO: This instruction may already have access group from
7607 // other pragmas e.g. #pragma clang loop vectorize. Append
7608 // so that the existing metadata is not overwritten.
7609 I.setMetadata(KindID: LLVMContext::MD_access_group, Node: AccessGroup);
7610 }
7611 }
7612}
7613
7614CanonicalLoopInfo *
7615OpenMPIRBuilder::fuseLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops) {
7616 CanonicalLoopInfo *firstLoop = Loops.front();
7617 CanonicalLoopInfo *lastLoop = Loops.back();
7618 Function *F = firstLoop->getPreheader()->getParent();
7619
7620 // Loop control blocks that will become orphaned later
7621 SmallVector<BasicBlock *> oldControlBBs;
7622 for (CanonicalLoopInfo *Loop : Loops)
7623 Loop->collectControlBlocks(BBs&: oldControlBBs);
7624
7625 // Collect original trip counts
7626 SmallVector<Value *> origTripCounts;
7627 for (CanonicalLoopInfo *L : Loops) {
7628 assert(L->isValid() && "All input loops must be valid canonical loops");
7629 origTripCounts.push_back(Elt: L->getTripCount());
7630 }
7631
7632 Builder.SetCurrentDebugLocation(DL);
7633
7634 // Compute max trip count.
7635 // The fused loop will be from 0 to max(origTripCounts)
7636 BasicBlock *TCBlock = BasicBlock::Create(Context&: F->getContext(), Name: "omp.fuse.comp.tc",
7637 Parent: F, InsertBefore: firstLoop->getHeader());
7638 Builder.SetInsertPoint(TCBlock);
7639 Value *fusedTripCount = nullptr;
7640 for (CanonicalLoopInfo *L : Loops) {
7641 assert(L->isValid() && "All loops to fuse must be valid canonical loops");
7642 Value *origTripCount = L->getTripCount();
7643 if (!fusedTripCount) {
7644 fusedTripCount = origTripCount;
7645 continue;
7646 }
7647 Value *condTP = Builder.CreateICmpSGT(LHS: fusedTripCount, RHS: origTripCount);
7648 fusedTripCount = Builder.CreateSelect(C: condTP, True: fusedTripCount, False: origTripCount,
7649 Name: ".omp.fuse.tc");
7650 }
7651
7652 // Generate new loop
7653 CanonicalLoopInfo *fused =
7654 createLoopSkeleton(DL, TripCount: fusedTripCount, F, PreInsertBefore: firstLoop->getBody(),
7655 PostInsertBefore: lastLoop->getLatch(), Name: "fused");
7656
7657 // Replace original loops with the fused loop
7658 // Preheader and After are not considered inside the CLI.
7659 // These are used to compute the individual TCs of the loops
7660 // so they have to be put before the resulting fused loop.
7661 // Moving them up for readability.
7662 for (size_t i = 0; i < Loops.size() - 1; ++i) {
7663 Loops[i]->getPreheader()->moveBefore(MovePos: TCBlock);
7664 Loops[i]->getAfter()->moveBefore(MovePos: TCBlock);
7665 }
7666 lastLoop->getPreheader()->moveBefore(MovePos: TCBlock);
7667
7668 for (size_t i = 0; i < Loops.size() - 1; ++i) {
7669 redirectTo(Source: Loops[i]->getPreheader(), Target: Loops[i]->getAfter(), DL);
7670 redirectTo(Source: Loops[i]->getAfter(), Target: Loops[i + 1]->getPreheader(), DL);
7671 }
7672 redirectTo(Source: lastLoop->getPreheader(), Target: TCBlock, DL);
7673 redirectTo(Source: TCBlock, Target: fused->getPreheader(), DL);
7674 redirectTo(Source: fused->getAfter(), Target: lastLoop->getAfter(), DL);
7675
7676 // Build the fused body
7677 // Create new Blocks with conditions that jump to the original loop bodies
7678 SmallVector<BasicBlock *> condBBs;
7679 SmallVector<Value *> condValues;
7680 for (size_t i = 0; i < Loops.size(); ++i) {
7681 BasicBlock *condBlock = BasicBlock::Create(
7682 Context&: F->getContext(), Name: "omp.fused.inner.cond", Parent: F, InsertBefore: Loops[i]->getBody());
7683 Builder.SetInsertPoint(condBlock);
7684 Value *condValue =
7685 Builder.CreateICmpSLT(LHS: fused->getIndVar(), RHS: origTripCounts[i]);
7686 condBBs.push_back(Elt: condBlock);
7687 condValues.push_back(Elt: condValue);
7688 }
7689 // Join the condition blocks with the bodies of the original loops
7690 redirectTo(Source: fused->getBody(), Target: condBBs[0], DL);
7691 for (size_t i = 0; i < Loops.size() - 1; ++i) {
7692 Builder.SetInsertPoint(condBBs[i]);
7693 Builder.CreateCondBr(Cond: condValues[i], True: Loops[i]->getBody(), False: condBBs[i + 1]);
7694 redirectAllPredecessorsTo(OldTarget: Loops[i]->getLatch(), NewTarget: condBBs[i + 1], DL);
7695 // Replace the IV with the fused IV
7696 Loops[i]->getIndVar()->replaceAllUsesWith(V: fused->getIndVar());
7697 }
7698 // Last body jumps to the created end body block
7699 Builder.SetInsertPoint(condBBs.back());
7700 Builder.CreateCondBr(Cond: condValues.back(), True: lastLoop->getBody(),
7701 False: fused->getLatch());
7702 redirectAllPredecessorsTo(OldTarget: lastLoop->getLatch(), NewTarget: fused->getLatch(), DL);
7703 // Replace the IV with the fused IV
7704 lastLoop->getIndVar()->replaceAllUsesWith(V: fused->getIndVar());
7705
7706 // The loop latch must have only one predecessor. Currently it is branched to
7707 // from both the last condition block and the last loop body
7708 fused->getLatch()->splitBasicBlockBefore(I: fused->getLatch()->begin(),
7709 BBName: "omp.fused.pre_latch");
7710
7711 // Remove unused parts
7712 removeUnusedBlocksFromParent(BBs: oldControlBBs);
7713
7714 // Invalidate old CLIs
7715 for (CanonicalLoopInfo *L : Loops)
7716 L->invalidate();
7717
7718#ifndef NDEBUG
7719 fused->assertOK();
7720#endif
7721 return fused;
7722}
7723
7724void OpenMPIRBuilder::addUnrollMetadata(CanonicalLoopInfo *Loop,
7725 ArrayRef<Metadata *> Properties) {
7726 assert(Loop->isValid() && "Expecting a valid CanonicalLoopInfo");
7727 if (Loop->VirtualCollapse)
7728 reportFatalInternalError(reason: "unrolling must precede Virtual collapse");
7729 addLoopMetadata(Loop, Properties);
7730}
7731
7732void OpenMPIRBuilder::unrollLoopFull(DebugLoc, CanonicalLoopInfo *Loop) {
7733 LLVMContext &Ctx = Builder.getContext();
7734 addUnrollMetadata(
7735 Loop, Properties: {MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.enable")),
7736 MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.full"))});
7737}
7738
7739void OpenMPIRBuilder::unrollLoopHeuristic(DebugLoc, CanonicalLoopInfo *Loop) {
7740 LLVMContext &Ctx = Builder.getContext();
7741 addUnrollMetadata(
7742 Loop, Properties: {
7743 MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.enable")),
7744 });
7745}
7746
7747void OpenMPIRBuilder::createIfVersion(CanonicalLoopInfo *CanonicalLoop,
7748 Value *IfCond, ValueToValueMapTy &VMap,
7749 LoopAnalysis &LIA, LoopInfo &LI, Loop *L,
7750 const Twine &NamePrefix) {
7751 Function *F = CanonicalLoop->getFunction();
7752
7753 // We can't do
7754 // if (cond) {
7755 // simd_loop;
7756 // } else {
7757 // non_simd_loop;
7758 // }
7759 // because then the CanonicalLoopInfo would only point to one of the loops:
7760 // leading to other constructs operating on the same loop to malfunction.
7761 // Instead generate
7762 // while (...) {
7763 // if (cond) {
7764 // simd_body;
7765 // } else {
7766 // not_simd_body;
7767 // }
7768 // }
7769 // At least for simple loops, LLVM seems able to hoist the if out of the loop
7770 // body at -O3
7771
7772 // Define where if branch should be inserted
7773 auto SplitBeforeIt = CanonicalLoop->getBody()->getFirstNonPHIIt();
7774
7775 // Create additional blocks for the if statement
7776 BasicBlock *Cond = SplitBeforeIt->getParent();
7777 llvm::LLVMContext &C = Cond->getContext();
7778 llvm::BasicBlock *ThenBlock = llvm::BasicBlock::Create(
7779 Context&: C, Name: NamePrefix + ".if.then", Parent: Cond->getParent(), InsertBefore: Cond->getNextNode());
7780 llvm::BasicBlock *ElseBlock = llvm::BasicBlock::Create(
7781 Context&: C, Name: NamePrefix + ".if.else", Parent: Cond->getParent(), InsertBefore: CanonicalLoop->getExit());
7782
7783 // Create if condition branch.
7784 Builder.SetInsertPoint(SplitBeforeIt);
7785 Instruction *BrInstr =
7786 Builder.CreateCondBr(Cond: IfCond, True: ThenBlock, /*ifFalse*/ False: ElseBlock);
7787 InsertPointTy IP(++BrInstr->getIterator());
7788 // Then block contains branch to omp loop body which needs to be vectorized
7789 spliceBB(IP, New: ThenBlock, CreateBranch: false, DL: Builder.getCurrentDebugLocation());
7790 ThenBlock->replaceSuccessorsPhiUsesWith(Old: Cond, New: ThenBlock);
7791
7792 Builder.SetInsertPoint(ElseBlock);
7793
7794 // Clone loop for the else branch
7795 SmallVector<BasicBlock *, 8> NewBlocks;
7796
7797 SmallVector<BasicBlock *, 8> ExistingBlocks;
7798 ExistingBlocks.reserve(N: L->getNumBlocks() + 1);
7799 ExistingBlocks.push_back(Elt: ThenBlock);
7800 ExistingBlocks.append(in_start: L->block_begin(), in_end: L->block_end());
7801 // Cond is the block that has the if clause condition
7802 // LoopCond is omp_loop.cond
7803 // LoopHeader is omp_loop.header
7804 BasicBlock *LoopCond = Cond->getUniquePredecessor();
7805 BasicBlock *LoopHeader = LoopCond->getUniquePredecessor();
7806 assert(LoopCond && LoopHeader && "Invalid loop structure");
7807 for (BasicBlock *Block : ExistingBlocks) {
7808 if (Block == L->getLoopPreheader() || Block == L->getLoopLatch() ||
7809 Block == LoopHeader || Block == LoopCond || Block == Cond) {
7810 continue;
7811 }
7812 BasicBlock *NewBB = CloneBasicBlock(BB: Block, VMap, NameSuffix: "", F);
7813
7814 // fix name not to be omp.if.then
7815 if (Block == ThenBlock)
7816 NewBB->setName(NamePrefix + ".if.else");
7817
7818 NewBB->moveBefore(MovePos: CanonicalLoop->getExit());
7819 VMap[Block] = NewBB;
7820 NewBlocks.push_back(Elt: NewBB);
7821 }
7822 remapInstructionsInBlocks(Blocks: NewBlocks, VMap);
7823 Builder.CreateBr(Dest: NewBlocks.front());
7824
7825 // The loop latch must have only one predecessor. Currently it is branched to
7826 // from both the 'then' and 'else' branches.
7827 L->getLoopLatch()->splitBasicBlockBefore(I: L->getLoopLatch()->begin(),
7828 BBName: NamePrefix + ".pre_latch");
7829
7830 // Ensure that the then block is added to the loop so we add the attributes in
7831 // the next step
7832 L->addBasicBlockToLoop(NewBB: ThenBlock, LI);
7833}
7834
7835unsigned
7836OpenMPIRBuilder::getOpenMPDefaultSimdAlign(const Triple &TargetTriple,
7837 const StringMap<bool> &Features) {
7838 if (TargetTriple.isX86()) {
7839 if (Features.lookup(Key: "avx512f"))
7840 return 512;
7841 else if (Features.lookup(Key: "avx"))
7842 return 256;
7843 return 128;
7844 }
7845 if (TargetTriple.isPPC())
7846 return 128;
7847 if (TargetTriple.isWasm())
7848 return 128;
7849 if (TargetTriple.isSystemZ())
7850 return 64;
7851 return 0;
7852}
7853
7854void OpenMPIRBuilder::applySimd(CanonicalLoopInfo *CanonicalLoop,
7855 MapVector<Value *, Value *> AlignedVars,
7856 Value *IfCond, OrderKind Order,
7857 ConstantInt *Simdlen, ConstantInt *Safelen) {
7858 if (CanonicalLoop->VirtualCollapse)
7859 CanonicalLoop->reportUnsupportedVirtualCollapse();
7860 LLVMContext &Ctx = Builder.getContext();
7861
7862 Function *F = CanonicalLoop->getFunction();
7863
7864 // Blocks must have terminators.
7865 // FIXME: Don't run analyses on incomplete/invalid IR.
7866 SmallVector<Instruction *> UIs;
7867 for (BasicBlock &BB : *F)
7868 if (!BB.hasTerminator())
7869 UIs.push_back(Elt: new UnreachableInst(F->getContext(), &BB));
7870
7871 // TODO: We should not rely on pass manager. Currently we use pass manager
7872 // only for getting llvm::Loop which corresponds to given CanonicalLoopInfo
7873 // object. We should have a method which returns all blocks between
7874 // CanonicalLoopInfo::getHeader() and CanonicalLoopInfo::getAfter()
7875 FunctionAnalysisManager FAM;
7876 FAM.registerPass(PassBuilder: []() { return DominatorTreeAnalysis(); });
7877 FAM.registerPass(PassBuilder: []() { return LoopAnalysis(); });
7878 FAM.registerPass(PassBuilder: []() { return PassInstrumentationAnalysis(); });
7879
7880 LoopAnalysis LIA;
7881 LoopInfo &&LI = LIA.run(F&: *F, AM&: FAM);
7882
7883 for (Instruction *I : UIs)
7884 I->eraseFromParent();
7885
7886 Loop *L = LI.getLoopFor(BB: CanonicalLoop->getHeader());
7887 if (AlignedVars.size()) {
7888 InsertPointTy IP = Builder.saveIP();
7889 for (auto &AlignedItem : AlignedVars) {
7890 Value *AlignedPtr = AlignedItem.first;
7891 Value *Alignment = AlignedItem.second;
7892 Instruction *loadInst = dyn_cast<Instruction>(Val: AlignedPtr);
7893 Builder.SetInsertPoint(loadInst->getNextNode());
7894 Builder.CreateAlignmentAssumption(DL: F->getDataLayout(), PtrValue: AlignedPtr,
7895 Alignment);
7896 }
7897 Builder.restoreIP(IP);
7898 }
7899
7900 if (IfCond) {
7901 ValueToValueMapTy VMap;
7902 createIfVersion(CanonicalLoop, IfCond, VMap, LIA, LI, L, NamePrefix: "simd");
7903 }
7904
7905 SmallPtrSet<BasicBlock *, 8> Reachable;
7906
7907 // Get the basic blocks from the loop in which memref instructions
7908 // can be found.
7909 // TODO: Generalize getting all blocks inside a CanonicalizeLoopInfo,
7910 // preferably without running any passes.
7911 for (BasicBlock *Block : L->getBlocks()) {
7912 if (Block == CanonicalLoop->getCond() ||
7913 Block == CanonicalLoop->getHeader())
7914 continue;
7915 Reachable.insert(Ptr: Block);
7916 }
7917
7918 SmallVector<Metadata *> LoopMDList;
7919
7920 // In presence of finite 'safelen', it may be unsafe to mark all
7921 // the memory instructions parallel, because loop-carried
7922 // dependences of 'safelen' iterations are possible.
7923 // If clause order(concurrent) is specified then the memory instructions
7924 // are marked parallel even if 'safelen' is finite.
7925 if ((Safelen == nullptr) || (Order == OrderKind::OMP_ORDER_concurrent))
7926 applyParallelAccessesMetadata(CLI: CanonicalLoop, Ctx, Loop: L, LoopInfo&: LI, LoopMDList);
7927
7928 // FIXME: the IF clause shares a loop backedge for the SIMD and non-SIMD
7929 // versions so we can't add the loop attributes in that case.
7930 if (IfCond) {
7931 // we can still add llvm.loop.parallel_access
7932 addLoopMetadata(Loop: CanonicalLoop, Properties: LoopMDList);
7933 return;
7934 }
7935
7936 // Use the above access group metadata to create loop level
7937 // metadata, which should be distinct for each loop.
7938 LoopMDList.push_back(
7939 Elt: MDNode::get(Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.vectorize.enable")}));
7940
7941 if (Simdlen || Safelen) {
7942 // If both simdlen and safelen clauses are specified, the value of the
7943 // simdlen parameter must be less than or equal to the value of the safelen
7944 // parameter. Therefore, use safelen only in the absence of simdlen.
7945 ConstantInt *VectorizeWidth = Simdlen == nullptr ? Safelen : Simdlen;
7946 LoopMDList.push_back(
7947 Elt: MDNode::get(Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.vectorize.width"),
7948 ConstantAsMetadata::get(C: VectorizeWidth)}));
7949 }
7950
7951 addLoopMetadata(Loop: CanonicalLoop, Properties: LoopMDList);
7952}
7953
7954void OpenMPIRBuilder::unrollLoopPartial(DebugLoc DL, CanonicalLoopInfo *Loop,
7955 int32_t Factor,
7956 CanonicalLoopInfo **UnrolledCLI) {
7957 assert(Factor >= 0 && "Unroll factor must not be negative");
7958
7959 Function *F = Loop->getFunction();
7960 LLVMContext &Ctx = F->getContext();
7961
7962 // If the unrolled loop is not used for another loop-associated directive, it
7963 // is sufficient to add metadata for the LoopUnrollPass.
7964 if (!UnrolledCLI) {
7965 SmallVector<Metadata *, 2> LoopMetadata;
7966 LoopMetadata.push_back(
7967 Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.enable")));
7968
7969 if (Factor >= 1) {
7970 ConstantAsMetadata *FactorConst = ConstantAsMetadata::get(
7971 C: ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: APInt(32, Factor)));
7972 LoopMetadata.push_back(Elt: MDNode::get(
7973 Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.count"), FactorConst}));
7974 }
7975
7976 addUnrollMetadata(Loop, Properties: LoopMetadata);
7977 return;
7978 }
7979
7980 if (Factor == 0) {
7981 if (Loop->VirtualCollapse)
7982 Loop->reportUnsupportedVirtualCollapse();
7983 Factor = omp::DefaultPartialUnrollFactor;
7984 }
7985
7986 // No change required with unroll factor 1.
7987 if (Factor == 1) {
7988 *UnrolledCLI = Loop;
7989 return;
7990 }
7991
7992 assert(Factor >= 2 &&
7993 "unrolling only makes sense with a factor of 2 or larger");
7994
7995 Type *IndVarTy = Loop->getIndVarType();
7996
7997 // Apply partial unrolling by tiling the loop by the unroll-factor, then fully
7998 // unroll the inner loop.
7999 Value *FactorVal =
8000 ConstantInt::get(Ty: IndVarTy, V: APInt(IndVarTy->getIntegerBitWidth(), Factor,
8001 /*isSigned=*/false));
8002 std::vector<CanonicalLoopInfo *> LoopNest =
8003 tileLoops(DL, Loops: {Loop}, TileSizes: {FactorVal});
8004 assert(LoopNest.size() == 2 && "Expect 2 loops after tiling");
8005 *UnrolledCLI = LoopNest[0];
8006 CanonicalLoopInfo *InnerLoop = LoopNest[1];
8007
8008 // LoopUnrollPass can only fully unroll loops with constant trip count.
8009 // Unroll by the unroll factor with a fallback epilog for the remainder
8010 // iterations if necessary.
8011 ConstantAsMetadata *FactorConst = ConstantAsMetadata::get(
8012 C: ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: APInt(32, Factor)));
8013 addLoopMetadata(
8014 Loop: InnerLoop,
8015 Properties: {MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.enable")),
8016 MDNode::get(
8017 Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.count"), FactorConst})});
8018
8019#ifndef NDEBUG
8020 (*UnrolledCLI)->assertOK();
8021#endif
8022}
8023
8024OpenMPIRBuilder::InsertPointTy
8025OpenMPIRBuilder::createCopyPrivate(const LocationDescription &Loc,
8026 llvm::Value *BufSize, llvm::Value *CpyBuf,
8027 llvm::Value *CpyFn, llvm::Value *DidIt) {
8028 if (!updateToLocation(Loc))
8029 return Loc.IP;
8030
8031 uint32_t SrcLocStrSize;
8032 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8033 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8034 Value *ThreadId = getOrCreateThreadID(Ident);
8035
8036 llvm::Value *DidItLD = Builder.CreateLoad(Ty: Builder.getInt32Ty(), Ptr: DidIt);
8037
8038 Value *Args[] = {Ident, ThreadId, BufSize, CpyBuf, CpyFn, DidItLD};
8039
8040 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_copyprivate);
8041 createRuntimeFunctionCall(Callee: Fn, Args);
8042
8043 return Builder.saveIP();
8044}
8045
8046OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createSingle(
8047 const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,
8048 FinalizeCallbackTy FiniCB, bool IsNowait, ArrayRef<llvm::Value *> CPVars,
8049 ArrayRef<llvm::Function *> CPFuncs) {
8050
8051 if (!updateToLocation(Loc))
8052 return Loc.IP;
8053
8054 // If needed allocate and initialize `DidIt` with 0.
8055 // DidIt: flag variable: 1=single thread; 0=not single thread.
8056 llvm::Value *DidIt = nullptr;
8057 if (!CPVars.empty()) {
8058 DidIt = Builder.CreateAlloca(Ty: llvm::Type::getInt32Ty(C&: Builder.getContext()));
8059 Builder.CreateStore(Val: Builder.getInt32(C: 0), Ptr: DidIt);
8060 }
8061
8062 Directive OMPD = Directive::OMPD_single;
8063 uint32_t SrcLocStrSize;
8064 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8065 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8066 Value *ThreadId = getOrCreateThreadID(Ident);
8067 Value *Args[] = {Ident, ThreadId};
8068
8069 Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_single);
8070 Instruction *EntryCall = createRuntimeFunctionCall(Callee: EntryRTLFn, Args);
8071
8072 Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_end_single);
8073 Instruction *ExitCall = createRuntimeFunctionCall(Callee: ExitRTLFn, Args);
8074
8075 auto FiniCBWrapper = [&](InsertPointTy IP) -> Error {
8076 if (Error Err = FiniCB(IP))
8077 return Err;
8078
8079 // The thread that executes the single region must set `DidIt` to 1.
8080 // This is used by __kmpc_copyprivate, to know if the caller is the
8081 // single thread or not.
8082 if (DidIt)
8083 Builder.CreateStore(Val: Builder.getInt32(C: 1), Ptr: DidIt);
8084
8085 return Error::success();
8086 };
8087
8088 // generates the following:
8089 // if (__kmpc_single()) {
8090 // .... single region ...
8091 // __kmpc_end_single
8092 // }
8093 // __kmpc_copyprivate
8094 // __kmpc_barrier
8095
8096 InsertPointOrErrorTy AfterIP =
8097 EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB: FiniCBWrapper,
8098 /*Conditional*/ true,
8099 /*hasFinalize*/ HasFinalize: true);
8100 if (!AfterIP)
8101 return AfterIP.takeError();
8102
8103 if (DidIt) {
8104 for (size_t I = 0, E = CPVars.size(); I < E; ++I)
8105 // NOTE BufSize is currently unused, so just pass 0.
8106 createCopyPrivate(Loc: LocationDescription(Builder.saveIP(), Loc.DL),
8107 /*BufSize=*/ConstantInt::get(Ty: Int64, V: 0), CpyBuf: CPVars[I],
8108 CpyFn: CPFuncs[I], DidIt);
8109 // NOTE __kmpc_copyprivate already inserts a barrier
8110 } else if (!IsNowait) {
8111 InsertPointOrErrorTy AfterIP =
8112 createBarrier(Loc: LocationDescription(Builder.saveIP(), Loc.DL),
8113 Kind: omp::Directive::OMPD_unknown, /* ForceSimpleCall */ false,
8114 /* CheckCancelFlag */ false);
8115 if (!AfterIP)
8116 return AfterIP.takeError();
8117 }
8118 return Builder.saveIP();
8119}
8120
8121OpenMPIRBuilder::InsertPointOrErrorTy
8122OpenMPIRBuilder::createScope(const LocationDescription &Loc,
8123 BodyGenCallbackTy BodyGenCB,
8124 FinalizeCallbackTy FiniCB, bool IsNowait) {
8125
8126 if (!updateToLocation(Loc))
8127 return Loc.IP;
8128
8129 // All threads execute the scope body — no conditional entry.
8130 InsertPointOrErrorTy AfterIP = EmitOMPInlinedRegion(
8131 OMPD: Directive::OMPD_scope, /*EntryCall=*/nullptr, /*ExitCall=*/nullptr,
8132 BodyGenCB, FiniCB, /*Conditional=*/false, /*HasFinalize=*/true,
8133 /*IsCancellable=*/false);
8134 if (!AfterIP)
8135 return AfterIP.takeError();
8136
8137 Builder.restoreIP(IP: *AfterIP);
8138 if (!IsNowait) {
8139 AfterIP = createBarrier(Loc: LocationDescription(Builder.saveIP(), Loc.DL),
8140 Kind: omp::Directive::OMPD_unknown,
8141 /*ForceSimpleCall=*/false,
8142 /*CheckCancelFlag=*/false);
8143 if (!AfterIP)
8144 return AfterIP.takeError();
8145 }
8146 return Builder.saveIP();
8147}
8148
8149OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createCritical(
8150 const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,
8151 FinalizeCallbackTy FiniCB, StringRef CriticalName, Value *HintInst) {
8152
8153 if (!updateToLocation(Loc))
8154 return Loc.IP;
8155
8156 Directive OMPD = Directive::OMPD_critical;
8157 uint32_t SrcLocStrSize;
8158 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8159 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8160 Value *ThreadId = getOrCreateThreadID(Ident);
8161 Value *LockVar = getOMPCriticalRegionLock(CriticalName);
8162 Value *Args[] = {Ident, ThreadId, LockVar};
8163
8164 SmallVector<llvm::Value *, 4> EnterArgs(std::begin(arr&: Args), std::end(arr&: Args));
8165 Function *RTFn = nullptr;
8166 if (HintInst) {
8167 // Add Hint to entry Args and create call
8168 EnterArgs.push_back(Elt: HintInst);
8169 RTFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_critical_with_hint);
8170 } else {
8171 RTFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_critical);
8172 }
8173 Instruction *EntryCall = createRuntimeFunctionCall(Callee: RTFn, Args: EnterArgs);
8174
8175 Function *ExitRTLFn =
8176 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_end_critical);
8177 Instruction *ExitCall = createRuntimeFunctionCall(Callee: ExitRTLFn, Args);
8178
8179 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
8180 /*Conditional*/ false, /*hasFinalize*/ HasFinalize: true);
8181}
8182
8183OpenMPIRBuilder::InsertPointTy
8184OpenMPIRBuilder::createOrderedDepend(const LocationDescription &Loc,
8185 InsertPointTy AllocaIP, unsigned NumLoops,
8186 ArrayRef<llvm::Value *> StoreValues,
8187 const Twine &Name, bool IsDependSource) {
8188 assert(
8189 llvm::all_of(StoreValues,
8190 [](Value *SV) { return SV->getType()->isIntegerTy(64); }) &&
8191 "OpenMP runtime requires depend vec with i64 type");
8192
8193 if (!updateToLocation(Loc))
8194 return Loc.IP;
8195
8196 // Allocate space for vector and generate alloc instruction.
8197 auto *ArrI64Ty = ArrayType::get(ElementType: Int64, NumElements: NumLoops);
8198 Builder.restoreIP(IP: AllocaIP);
8199 AllocaInst *ArgsBase = Builder.CreateAlloca(Ty: ArrI64Ty, ArraySize: nullptr, Name);
8200 ArgsBase->setAlignment(Align(8));
8201 updateToLocation(Loc);
8202
8203 // Store the index value with offset in depend vector.
8204 for (unsigned I = 0; I < NumLoops; ++I) {
8205 Value *DependAddrGEPIter = Builder.CreateInBoundsGEP(
8206 Ty: ArrI64Ty, Ptr: ArgsBase, IdxList: {Builder.getInt64(C: 0), Builder.getInt64(C: I)});
8207 StoreInst *STInst = Builder.CreateStore(Val: StoreValues[I], Ptr: DependAddrGEPIter);
8208 STInst->setAlignment(Align(8));
8209 }
8210
8211 Value *DependBaseAddrGEP = Builder.CreateInBoundsGEP(
8212 Ty: ArrI64Ty, Ptr: ArgsBase, IdxList: {Builder.getInt64(C: 0), Builder.getInt64(C: 0)});
8213
8214 uint32_t SrcLocStrSize;
8215 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8216 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8217 Value *ThreadId = getOrCreateThreadID(Ident);
8218 Value *Args[] = {Ident, ThreadId, DependBaseAddrGEP};
8219
8220 Function *RTLFn = nullptr;
8221 if (IsDependSource)
8222 RTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_doacross_post);
8223 else
8224 RTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_doacross_wait);
8225 createRuntimeFunctionCall(Callee: RTLFn, Args);
8226
8227 return Builder.saveIP();
8228}
8229
8230OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createOrderedThreadsSimd(
8231 const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,
8232 FinalizeCallbackTy FiniCB, bool IsThreads) {
8233 if (!updateToLocation(Loc))
8234 return Loc.IP;
8235
8236 Directive OMPD = Directive::OMPD_ordered_blockassoc;
8237 Instruction *EntryCall = nullptr;
8238 Instruction *ExitCall = nullptr;
8239
8240 if (IsThreads) {
8241 uint32_t SrcLocStrSize;
8242 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8243 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8244 Value *ThreadId = getOrCreateThreadID(Ident);
8245 Value *Args[] = {Ident, ThreadId};
8246
8247 Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_ordered);
8248 EntryCall = createRuntimeFunctionCall(Callee: EntryRTLFn, Args);
8249
8250 Function *ExitRTLFn =
8251 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_end_ordered);
8252 ExitCall = createRuntimeFunctionCall(Callee: ExitRTLFn, Args);
8253 }
8254
8255 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
8256 /*Conditional*/ false, /*hasFinalize*/ HasFinalize: true);
8257}
8258
8259OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::EmitOMPInlinedRegion(
8260 Directive OMPD, Instruction *EntryCall, Instruction *ExitCall,
8261 BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB, bool Conditional,
8262 bool HasFinalize, bool IsCancellable) {
8263
8264 if (HasFinalize)
8265 FinalizationStack.push_back(Elt: {FiniCB, OMPD, IsCancellable});
8266
8267 // Create inlined region's entry and body blocks, in preparation
8268 // for conditional creation
8269 BasicBlock *EntryBB = Builder.GetInsertBlock();
8270 Instruction *SplitPos = EntryBB->getTerminatorOrNull();
8271 if (!isa_and_nonnull<UncondBrInst, CondBrInst>(Val: SplitPos)) {
8272 SplitPos = new UnreachableInst(Builder.getContext(), EntryBB);
8273 SplitPos->setDebugLoc(Builder.getCurrentDebugLocation());
8274 }
8275 BasicBlock *ExitBB = EntryBB->splitBasicBlock(I: SplitPos, BBName: "omp_region.end");
8276 BasicBlock *FiniBB =
8277 EntryBB->splitBasicBlock(I: EntryBB->getTerminator(), BBName: "omp_region.finalize");
8278
8279 Builder.SetInsertPoint(EntryBB->getTerminator());
8280 emitCommonDirectiveEntry(OMPD, EntryCall, ExitBB, Conditional);
8281
8282 // generate body
8283 if (Error Err =
8284 BodyGenCB(/* AllocaIP */ InsertPointTy(),
8285 /* CodeGenIP */ Builder.saveIP(), /* DeallocBlocks */ {}))
8286 return Err;
8287
8288 // emit exit call and do any needed finalization.
8289 auto FinIP = FiniBB->getFirstInsertionPt();
8290 assert(FiniBB->getTerminator()->getNumSuccessors() == 1 &&
8291 FiniBB->getTerminator()->getSuccessor(0) == ExitBB &&
8292 "Unexpected control flow graph state!!");
8293 InsertPointOrErrorTy AfterIP =
8294 emitCommonDirectiveExit(OMPD, FinIP, ExitCall, HasFinalize);
8295 if (!AfterIP)
8296 return AfterIP.takeError();
8297
8298 // If we are skipping the region of a non conditional, remove the exit
8299 // block, and clear the builder's insertion point.
8300 assert(SplitPos->getParent() == ExitBB &&
8301 "Unexpected Insertion point location!");
8302 auto merged = MergeBlockIntoPredecessor(BB: ExitBB);
8303 BasicBlock *ExitPredBB = SplitPos->getParent();
8304 auto InsertBB = merged ? ExitPredBB : ExitBB;
8305 if (!isa_and_nonnull<UncondBrInst, CondBrInst>(Val: SplitPos))
8306 SplitPos->eraseFromParent();
8307 Builder.SetInsertPoint(InsertBB);
8308
8309 return Builder.saveIP();
8310}
8311
8312OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitCommonDirectiveEntry(
8313 Directive OMPD, Value *EntryCall, BasicBlock *ExitBB, bool Conditional) {
8314 // if nothing to do, Return current insertion point.
8315 if (!Conditional || !EntryCall)
8316 return Builder.saveIP();
8317
8318 BasicBlock *EntryBB = Builder.GetInsertBlock();
8319 Value *CallBool = Builder.CreateIsNotNull(Arg: EntryCall);
8320 auto *ThenBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp_region.body");
8321 auto *UI = new UnreachableInst(Builder.getContext(), ThenBB);
8322
8323 // Emit thenBB and set the Builder's insertion point there for
8324 // body generation next. Place the block after the current block.
8325 Function *CurFn = EntryBB->getParent();
8326 CurFn->insert(Position: std::next(x: EntryBB->getIterator()), BB: ThenBB);
8327
8328 // Move Entry branch to end of ThenBB, and replace with conditional
8329 // branch (If-stmt)
8330 Instruction *EntryBBTI = EntryBB->getTerminator();
8331 Builder.CreateCondBr(Cond: CallBool, True: ThenBB, False: ExitBB);
8332 EntryBBTI->removeFromParent();
8333 Builder.SetInsertPoint(UI);
8334 Builder.Insert(I: EntryBBTI);
8335 UI->eraseFromParent();
8336 Builder.SetInsertPoint(ThenBB->getTerminator());
8337
8338 // return an insertion point to ExitBB.
8339 return ExitBB->getFirstInsertionPt();
8340}
8341
8342OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitCommonDirectiveExit(
8343 omp::Directive OMPD, InsertPointTy FinIP, Instruction *ExitCall,
8344 bool HasFinalize) {
8345
8346 Builder.restoreIP(IP: FinIP);
8347
8348 // If there is finalization to do, emit it before the exit call
8349 if (HasFinalize) {
8350 assert(!FinalizationStack.empty() &&
8351 "Unexpected finalization stack state!");
8352
8353 FinalizationInfo Fi = FinalizationStack.pop_back_val();
8354 assert(Fi.DK == OMPD && "Unexpected Directive for Finalization call!");
8355
8356 BasicBlock *FinBB = FinIP.getNodeParent();
8357 if (Error Err = Fi.mergeFiniBB(Builder, OtherFiniBB: FinBB))
8358 return std::move(Err);
8359
8360 // Exit condition: insertion point is before the terminator of the new Fini
8361 // block
8362 Builder.SetInsertPoint(FinBB->getTerminator());
8363 }
8364
8365 if (!ExitCall)
8366 return Builder.saveIP();
8367
8368 // place the Exitcall as last instruction before Finalization block terminator
8369 ExitCall->removeFromParent();
8370 Builder.Insert(I: ExitCall);
8371
8372 return ExitCall->getIterator();
8373}
8374
8375OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createCopyinClauseBlocks(
8376 InsertPointTy IP, Value *MasterAddr, Value *PrivateAddr,
8377 llvm::IntegerType *IntPtrTy, bool BranchtoEnd) {
8378 if (!IP.isValid())
8379 return IP;
8380
8381 IRBuilder<>::InsertPointGuard IPG(Builder);
8382
8383 // creates the following CFG structure
8384 // OMP_Entry : (MasterAddr != PrivateAddr)?
8385 // F T
8386 // | \
8387 // | copin.not.master
8388 // | /
8389 // v /
8390 // copyin.not.master.end
8391 // |
8392 // v
8393 // OMP.Entry.Next
8394
8395 BasicBlock *OMP_Entry = IP.getNodeParent();
8396 Function *CurFn = OMP_Entry->getParent();
8397 BasicBlock *CopyBegin =
8398 BasicBlock::Create(Context&: M.getContext(), Name: "copyin.not.master", Parent: CurFn);
8399 BasicBlock *CopyEnd = nullptr;
8400
8401 // If entry block is terminated, split to preserve the branch to following
8402 // basic block (i.e. OMP.Entry.Next), otherwise, leave everything as is.
8403 if (isa_and_nonnull<CondBrInst>(Val: OMP_Entry->getTerminatorOrNull())) {
8404 CopyEnd = OMP_Entry->splitBasicBlock(I: OMP_Entry->getTerminator(),
8405 BBName: "copyin.not.master.end");
8406 OMP_Entry->getTerminator()->eraseFromParent();
8407 } else {
8408 CopyEnd =
8409 BasicBlock::Create(Context&: M.getContext(), Name: "copyin.not.master.end", Parent: CurFn);
8410 }
8411
8412 Builder.SetInsertPoint(OMP_Entry);
8413 Value *MasterPtr = Builder.CreatePtrToInt(V: MasterAddr, DestTy: IntPtrTy);
8414 Value *PrivatePtr = Builder.CreatePtrToInt(V: PrivateAddr, DestTy: IntPtrTy);
8415 Value *cmp = Builder.CreateICmpNE(LHS: MasterPtr, RHS: PrivatePtr);
8416 Builder.CreateCondBr(Cond: cmp, True: CopyBegin, False: CopyEnd);
8417
8418 Builder.SetInsertPoint(CopyBegin);
8419 if (BranchtoEnd)
8420 Builder.SetInsertPoint(Builder.CreateBr(Dest: CopyEnd));
8421
8422 return Builder.saveIP();
8423}
8424
8425CallInst *OpenMPIRBuilder::createOMPAlloc(const LocationDescription &Loc,
8426 Value *Size, Value *Allocator,
8427 std::string Name) {
8428 IRBuilder<>::InsertPointGuard IPG(Builder);
8429 if (!updateToLocation(Loc))
8430 return nullptr;
8431
8432 uint32_t SrcLocStrSize;
8433 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8434 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8435 Value *ThreadId = getOrCreateThreadID(Ident);
8436 Value *Args[] = {ThreadId, Size, Allocator};
8437
8438 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_alloc);
8439
8440 return createRuntimeFunctionCall(Callee: Fn, Args, Name);
8441}
8442
8443CallInst *OpenMPIRBuilder::createOMPAlignedAlloc(const LocationDescription &Loc,
8444 Value *Align, Value *Size,
8445 Value *Allocator,
8446 std::string Name) {
8447 IRBuilder<>::InsertPointGuard IPG(Builder);
8448 if (!updateToLocation(Loc))
8449 return nullptr;
8450
8451 uint32_t SrcLocStrSize;
8452 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8453 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8454 Value *ThreadId = getOrCreateThreadID(Ident);
8455 Value *Args[] = {ThreadId, Align, Size, Allocator};
8456
8457 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_aligned_alloc);
8458
8459 return Builder.CreateCall(Callee: Fn, Args, Name);
8460}
8461
8462CallInst *OpenMPIRBuilder::createOMPFree(const LocationDescription &Loc,
8463 Value *Addr, Value *Allocator,
8464 std::string Name) {
8465 IRBuilder<>::InsertPointGuard IPG(Builder);
8466 if (!updateToLocation(Loc))
8467 return nullptr;
8468
8469 uint32_t SrcLocStrSize;
8470 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8471 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8472 Value *ThreadId = getOrCreateThreadID(Ident);
8473 Value *Args[] = {ThreadId, Addr, Allocator};
8474 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_free);
8475 return createRuntimeFunctionCall(Callee: Fn, Args, Name);
8476}
8477
8478CallInst *OpenMPIRBuilder::createOMPAllocShared(const LocationDescription &Loc,
8479 Value *Size,
8480 const Twine &Name) {
8481 IRBuilder<>::InsertPointGuard IPG(Builder);
8482 updateToLocation(Loc);
8483
8484 Value *Args[] = {Size};
8485 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_alloc_shared);
8486 CallInst *Call = Builder.CreateCall(Callee: Fn, Args, Name);
8487 Call->addRetAttr(Attr: Attribute::getWithAlignment(
8488 Context&: M.getContext(), Alignment: M.getDataLayout().getPrefTypeAlign(Ty: Int64)));
8489 return Call;
8490}
8491
8492CallInst *OpenMPIRBuilder::createOMPAllocShared(const LocationDescription &Loc,
8493 Type *VarType,
8494 const Twine &Name) {
8495 return createOMPAllocShared(
8496 Loc, Size: Builder.getInt64(C: M.getDataLayout().getTypeAllocSize(Ty: VarType)), Name);
8497}
8498
8499CallInst *OpenMPIRBuilder::createOMPFreeShared(const LocationDescription &Loc,
8500 Value *Addr, Value *Size,
8501 const Twine &Name) {
8502 IRBuilder<>::InsertPointGuard IPG(Builder);
8503 updateToLocation(Loc);
8504
8505 Value *Args[] = {Addr, Size};
8506 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_free_shared);
8507 return Builder.CreateCall(Callee: Fn, Args, Name);
8508}
8509
8510CallInst *OpenMPIRBuilder::createOMPFreeShared(const LocationDescription &Loc,
8511 Value *Addr, Type *VarType,
8512 const Twine &Name) {
8513 return createOMPFreeShared(
8514 Loc, Addr, Size: Builder.getInt64(C: M.getDataLayout().getTypeAllocSize(Ty: VarType)),
8515 Name);
8516}
8517
8518CallInst *OpenMPIRBuilder::createOMPInteropInit(
8519 const LocationDescription &Loc, Value *InteropVar,
8520 omp::OMPInteropType InteropType, Value *Device, Value *NumDependences,
8521 Value *DependenceAddress, bool HaveNowaitClause) {
8522 IRBuilder<>::InsertPointGuard IPG(Builder);
8523 updateToLocation(Loc);
8524
8525 uint32_t SrcLocStrSize;
8526 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8527 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8528 Value *ThreadId = getOrCreateThreadID(Ident);
8529 if (Device == nullptr)
8530 Device = Constant::getAllOnesValue(Ty: Int32);
8531 else if (Device->getType() != Int32)
8532 Device = Builder.CreateIntCast(V: Device, DestTy: Int32, /*isSigned=*/true);
8533 Constant *InteropTypeVal = ConstantInt::get(Ty: Int32, V: (int)InteropType);
8534 if (NumDependences == nullptr) {
8535 NumDependences = ConstantInt::get(Ty: Int32, V: 0);
8536 PointerType *PointerTypeVar = PointerType::getUnqual(C&: M.getContext());
8537 DependenceAddress = ConstantPointerNull::get(T: PointerTypeVar);
8538 }
8539 Value *HaveNowaitClauseVal = ConstantInt::get(Ty: Int32, V: HaveNowaitClause);
8540 Value *Args[] = {
8541 Ident, ThreadId, InteropVar, InteropTypeVal,
8542 Device, NumDependences, DependenceAddress, HaveNowaitClauseVal};
8543
8544 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___tgt_interop_init);
8545
8546 return createRuntimeFunctionCall(Callee: Fn, Args);
8547}
8548
8549CallInst *OpenMPIRBuilder::createOMPInteropDestroy(
8550 const LocationDescription &Loc, Value *InteropVar, Value *Device,
8551 Value *NumDependences, Value *DependenceAddress, bool HaveNowaitClause) {
8552 IRBuilder<>::InsertPointGuard IPG(Builder);
8553 updateToLocation(Loc);
8554
8555 uint32_t SrcLocStrSize;
8556 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8557 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8558 Value *ThreadId = getOrCreateThreadID(Ident);
8559 if (Device == nullptr)
8560 Device = Constant::getAllOnesValue(Ty: Int32);
8561 else if (Device->getType() != Int32)
8562 Device = Builder.CreateIntCast(V: Device, DestTy: Int32, /*isSigned=*/true);
8563 if (NumDependences == nullptr) {
8564 NumDependences = ConstantInt::get(Ty: Int32, V: 0);
8565 PointerType *PointerTypeVar = PointerType::getUnqual(C&: M.getContext());
8566 DependenceAddress = ConstantPointerNull::get(T: PointerTypeVar);
8567 }
8568 Value *HaveNowaitClauseVal = ConstantInt::get(Ty: Int32, V: HaveNowaitClause);
8569 Value *Args[] = {
8570 Ident, ThreadId, InteropVar, Device,
8571 NumDependences, DependenceAddress, HaveNowaitClauseVal};
8572
8573 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___tgt_interop_destroy);
8574
8575 return createRuntimeFunctionCall(Callee: Fn, Args);
8576}
8577
8578CallInst *OpenMPIRBuilder::createOMPInteropUse(const LocationDescription &Loc,
8579 Value *InteropVar, Value *Device,
8580 Value *NumDependences,
8581 Value *DependenceAddress,
8582 bool HaveNowaitClause) {
8583 IRBuilder<>::InsertPointGuard IPG(Builder);
8584 updateToLocation(Loc);
8585 uint32_t SrcLocStrSize;
8586 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8587 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8588 Value *ThreadId = getOrCreateThreadID(Ident);
8589 if (Device == nullptr)
8590 Device = Constant::getAllOnesValue(Ty: Int32);
8591 else if (Device->getType() != Int32)
8592 Device = Builder.CreateIntCast(V: Device, DestTy: Int32, /*isSigned=*/true);
8593 if (NumDependences == nullptr) {
8594 NumDependences = ConstantInt::get(Ty: Int32, V: 0);
8595 PointerType *PointerTypeVar = PointerType::getUnqual(C&: M.getContext());
8596 DependenceAddress = ConstantPointerNull::get(T: PointerTypeVar);
8597 }
8598 Value *HaveNowaitClauseVal = ConstantInt::get(Ty: Int32, V: HaveNowaitClause);
8599 Value *Args[] = {
8600 Ident, ThreadId, InteropVar, Device,
8601 NumDependences, DependenceAddress, HaveNowaitClauseVal};
8602
8603 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___tgt_interop_use);
8604
8605 return createRuntimeFunctionCall(Callee: Fn, Args);
8606}
8607
8608CallInst *OpenMPIRBuilder::createCachedThreadPrivate(
8609 const LocationDescription &Loc, llvm::Value *Pointer,
8610 llvm::ConstantInt *Size, const llvm::Twine &Name) {
8611 IRBuilder<>::InsertPointGuard IPG(Builder);
8612 updateToLocation(Loc);
8613
8614 uint32_t SrcLocStrSize;
8615 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8616 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8617 Value *ThreadId = getOrCreateThreadID(Ident);
8618 Constant *ThreadPrivateCache =
8619 getOrCreateInternalVariable(Ty: Int8PtrPtr, Name: Name.str());
8620 llvm::Value *Args[] = {Ident, ThreadId, Pointer, Size, ThreadPrivateCache};
8621
8622 Function *Fn =
8623 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_threadprivate_cached);
8624
8625 return createRuntimeFunctionCall(Callee: Fn, Args);
8626}
8627
8628Constant *OpenMPIRBuilder::emitKernelEnvironment(
8629 const LocationDescription &Loc,
8630 const llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs &Attrs) {
8631 assert(!Attrs.MaxThreads.empty() && !Attrs.MaxTeams.empty() &&
8632 "expected num_threads and num_teams to be specified");
8633
8634 if (!updateToLocation(Loc))
8635 return nullptr;
8636
8637 uint32_t SrcLocStrSize;
8638 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8639 Constant *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8640 Constant *IsSPMDVal = ConstantInt::getSigned(Ty: Int8, V: Attrs.ExecFlags);
8641 Constant *UseGenericStateMachineVal = ConstantInt::getSigned(
8642 Ty: Int8, V: Attrs.ExecFlags != omp::OMP_TGT_EXEC_MODE_SPMD &&
8643 Attrs.ExecFlags != omp::OMP_TGT_EXEC_MODE_SPMD_NO_LOOP);
8644 Constant *MayUseNestedParallelismVal = ConstantInt::getSigned(Ty: Int8, V: true);
8645 Constant *DebugIndentionLevelVal = ConstantInt::getSigned(Ty: Int16, V: 0);
8646
8647 Function *DebugKernelWrapper = Builder.GetInsertBlock()->getParent();
8648 Function *Kernel = DebugKernelWrapper;
8649
8650 // We need to strip the debug prefix to get the correct kernel name.
8651 StringRef KernelName = Kernel->getName();
8652 const std::string DebugPrefix = "_debug__";
8653 if (KernelName.ends_with(Suffix: DebugPrefix)) {
8654 KernelName = KernelName.drop_back(N: DebugPrefix.length());
8655 Kernel = M.getFunction(Name: KernelName);
8656 assert(Kernel && "Expected the real kernel to exist");
8657 }
8658
8659 // Manifest the launch configuration in the metadata matching the kernel
8660 // environment.
8661 if (Attrs.MinTeams.front() > 1 || Attrs.MaxTeams.front() > 0)
8662 writeTeamsForKernel(T, Kernel&: *Kernel, LB: Attrs.MinTeams.front(),
8663 UB: Attrs.MaxTeams.front());
8664
8665 // Don't derive or write thread bounds for Bare kernels.
8666 int32_t MaxThreadsVal = Attrs.MaxThreads.front();
8667 if (Attrs.ExecFlags != omp::OMP_TGT_EXEC_MODE_BARE) {
8668 // If MaxThreads is not set and needs adjustment, select the maximum
8669 // between the default workgroup size and the MinThreads value. This is
8670 // only meaningful for targets with a known grid value (i.e. GPUs); for
8671 // other targets (e.g. host kernels) leave it unset so the runtime falls
8672 // back to its own device-specific default.
8673 if (MaxThreadsVal < 0 && UseDefaultMaxThreads && hasGridValue(T))
8674 MaxThreadsVal =
8675 std::max(a: int32_t(getGridValue(T, Kernel).GV_Default_WG_Size),
8676 b: Attrs.MinThreads.front());
8677
8678 // Generic mode runs the main thread on a warp of its own, past
8679 // thread_limit. Reserve the widest warp any target has. Not on SPIR-V,
8680 // causes problems with Level Zero.
8681 if (MaxThreadsVal > 0 &&
8682 Attrs.ExecFlags == omp::OMP_TGT_EXEC_MODE_GENERIC && hasGridValue(T) &&
8683 !T.isSPIRV())
8684 MaxThreadsVal = int32_t(
8685 std::min<int64_t>(a: int64_t(MaxThreadsVal) + 64,
8686 b: int64_t(getGridValue(T, Kernel).GV_Max_WG_Size)));
8687
8688 if (MaxThreadsVal > 0)
8689 writeThreadBoundsForKernel(T, Kernel&: *Kernel, LB: Attrs.MinThreads.front(),
8690 UB: MaxThreadsVal);
8691 }
8692
8693 Constant *MinThreads =
8694 ConstantInt::getSigned(Ty: Int32, V: Attrs.MinThreads.front());
8695 Constant *MaxThreads = ConstantInt::getSigned(Ty: Int32, V: MaxThreadsVal);
8696 Constant *MinTeams = ConstantInt::getSigned(Ty: Int32, V: Attrs.MinTeams.front());
8697 Constant *MaxTeams = ConstantInt::getSigned(Ty: Int32, V: Attrs.MaxTeams.front());
8698 Constant *ReductionDataSize =
8699 ConstantInt::getSigned(Ty: Int32, V: Attrs.ReductionDataSize);
8700
8701 const DataLayout &DL = M.getDataLayout();
8702
8703 Twine DynamicEnvironmentName = KernelName + "_dynamic_environment";
8704 Constant *DynamicEnvironmentInitializer =
8705 ConstantStruct::get(T: DynamicEnvironment, V: {DebugIndentionLevelVal});
8706 GlobalVariable *DynamicEnvironmentGV = new GlobalVariable(
8707 M, DynamicEnvironment, /*IsConstant=*/false, GlobalValue::WeakODRLinkage,
8708 DynamicEnvironmentInitializer, DynamicEnvironmentName,
8709 /*InsertBefore=*/nullptr, GlobalValue::NotThreadLocal,
8710 DL.getDefaultGlobalsAddressSpace());
8711 DynamicEnvironmentGV->setVisibility(GlobalValue::ProtectedVisibility);
8712
8713 Constant *DynamicEnvironment =
8714 DynamicEnvironmentGV->getType() == DynamicEnvironmentPtr
8715 ? DynamicEnvironmentGV
8716 : ConstantExpr::getAddrSpaceCast(C: DynamicEnvironmentGV,
8717 Ty: DynamicEnvironmentPtr);
8718
8719 Constant *ConfigurationEnvironmentInitializer = ConstantStruct::get(
8720 T: ConfigurationEnvironment, V: {
8721 UseGenericStateMachineVal,
8722 MayUseNestedParallelismVal,
8723 IsSPMDVal,
8724 MinThreads,
8725 MaxThreads,
8726 MinTeams,
8727 MaxTeams,
8728 ReductionDataSize,
8729 });
8730 Constant *KernelEnvironmentInitializer = ConstantStruct::get(
8731 T: KernelEnvironment, V: {
8732 ConfigurationEnvironmentInitializer,
8733 Ident,
8734 DynamicEnvironment,
8735 });
8736 std::string KernelEnvironmentName =
8737 (KernelName + "_kernel_environment").str();
8738 GlobalVariable *KernelEnvironmentGV = new GlobalVariable(
8739 M, KernelEnvironment, /*IsConstant=*/true, GlobalValue::WeakODRLinkage,
8740 KernelEnvironmentInitializer, KernelEnvironmentName,
8741 /*InsertBefore=*/nullptr, GlobalValue::NotThreadLocal,
8742 DL.getDefaultGlobalsAddressSpace());
8743 KernelEnvironmentGV->setVisibility(GlobalValue::ProtectedVisibility);
8744
8745 return KernelEnvironmentGV->getType() == KernelEnvironmentPtr
8746 ? KernelEnvironmentGV
8747 : ConstantExpr::getAddrSpaceCast(C: KernelEnvironmentGV,
8748 Ty: KernelEnvironmentPtr);
8749}
8750
8751OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTargetInit(
8752 const LocationDescription &Loc,
8753 const llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs &Attrs) {
8754 Constant *KernelEnvironment = emitKernelEnvironment(Loc, Attrs);
8755 if (!KernelEnvironment)
8756 return Loc.IP;
8757
8758 if (!updateToLocation(Loc))
8759 return Loc.IP;
8760
8761 Function *DebugKernelWrapper = Builder.GetInsertBlock()->getParent();
8762 Function *Fn = getOrCreateRuntimeFunctionPtr(
8763 FnID: omp::RuntimeFunction::OMPRTL___kmpc_target_init);
8764
8765 Value *KernelLaunchEnvironment =
8766 DebugKernelWrapper->getArg(i: DebugKernelWrapper->arg_size() - 1);
8767 Type *KernelLaunchEnvParamTy = Fn->getFunctionType()->getParamType(i: 1);
8768 KernelLaunchEnvironment =
8769 KernelLaunchEnvironment->getType() == KernelLaunchEnvParamTy
8770 ? KernelLaunchEnvironment
8771 : Builder.CreateAddrSpaceCast(V: KernelLaunchEnvironment,
8772 DestTy: KernelLaunchEnvParamTy);
8773 CallInst *ThreadKind = createRuntimeFunctionCall(
8774 Callee: Fn, Args: {KernelEnvironment, KernelLaunchEnvironment});
8775
8776 Value *ExecUserCode = Builder.CreateICmpEQ(
8777 LHS: ThreadKind, RHS: Constant::getAllOnesValue(Ty: ThreadKind->getType()),
8778 Name: "exec_user_code");
8779
8780 // ThreadKind = __kmpc_target_init(...)
8781 // if (ThreadKind == -1)
8782 // user_code
8783 // else
8784 // return;
8785
8786 auto *UI = Builder.CreateUnreachable();
8787 BasicBlock *CheckBB = UI->getParent();
8788 BasicBlock *UserCodeEntryBB = CheckBB->splitBasicBlock(I: UI, BBName: "user_code.entry");
8789
8790 BasicBlock *WorkerExitBB = BasicBlock::Create(
8791 Context&: CheckBB->getContext(), Name: "worker.exit", Parent: CheckBB->getParent());
8792 Builder.SetInsertPoint(WorkerExitBB);
8793 Builder.CreateRetVoid();
8794
8795 auto *CheckBBTI = CheckBB->getTerminator();
8796 Builder.SetInsertPoint(CheckBBTI);
8797 Builder.CreateCondBr(Cond: ExecUserCode, True: UI->getParent(), False: WorkerExitBB);
8798
8799 CheckBBTI->eraseFromParent();
8800 UI->eraseFromParent();
8801
8802 // Continue in the "user_code" block, see diagram above and in
8803 // openmp/libomptarget/deviceRTLs/common/include/target.h .
8804 return UserCodeEntryBB->getFirstInsertionPt();
8805}
8806
8807void OpenMPIRBuilder::createTargetDeinit(const LocationDescription &Loc,
8808 int32_t TeamsReductionDataSize) {
8809 if (!updateToLocation(Loc))
8810 return;
8811
8812 Function *Fn = getOrCreateRuntimeFunctionPtr(
8813 FnID: omp::RuntimeFunction::OMPRTL___kmpc_target_deinit);
8814
8815 createRuntimeFunctionCall(Callee: Fn, Args: {});
8816
8817 if (!TeamsReductionDataSize)
8818 return;
8819
8820 Function *Kernel = Builder.GetInsertBlock()->getParent();
8821 // We need to strip the debug prefix to get the correct kernel name.
8822 StringRef KernelName = Kernel->getName();
8823 const std::string DebugPrefix = "_debug__";
8824 if (KernelName.ends_with(Suffix: DebugPrefix))
8825 KernelName = KernelName.drop_back(N: DebugPrefix.length());
8826 auto *KernelEnvironmentGV =
8827 M.getNamedGlobal(Name: (KernelName + "_kernel_environment").str());
8828 assert(KernelEnvironmentGV && "Expected kernel environment global\n");
8829 auto *KernelEnvironmentInitializer = KernelEnvironmentGV->getInitializer();
8830 auto *NewInitializer = ConstantFoldInsertValueInstruction(
8831 Agg: KernelEnvironmentInitializer,
8832 Val: ConstantInt::get(Ty: Int32, V: TeamsReductionDataSize), Idxs: {0, 7});
8833 KernelEnvironmentGV->setInitializer(NewInitializer);
8834}
8835
8836static void updateNVPTXAttr(Function &Kernel, StringRef Name, int32_t Value,
8837 bool Min) {
8838 if (Kernel.hasFnAttribute(Kind: Name)) {
8839 int32_t OldLimit = Kernel.getFnAttributeAsParsedInteger(Kind: Name);
8840 Value = Min ? std::min(a: OldLimit, b: Value) : std::max(a: OldLimit, b: Value);
8841 }
8842 Kernel.addFnAttr(Kind: Name, Val: llvm::utostr(X: Value));
8843}
8844
8845std::pair<int32_t, int32_t>
8846OpenMPIRBuilder::readThreadBoundsForKernel(const Triple &T, Function &Kernel) {
8847 int32_t ThreadLimit =
8848 Kernel.getFnAttributeAsParsedInteger(Kind: "omp_target_thread_limit");
8849
8850 if (T.isAMDGPU()) {
8851 const auto &Attr = Kernel.getFnAttribute(Kind: "amdgpu-flat-work-group-size");
8852 if (!Attr.isValid() || !Attr.isStringAttribute())
8853 return {0, ThreadLimit};
8854 auto [LBStr, UBStr] = Attr.getValueAsString().split(Separator: ',');
8855 int32_t LB, UB;
8856 if (!llvm::to_integer(S: UBStr, Num&: UB, Base: 10))
8857 return {0, ThreadLimit};
8858 UB = ThreadLimit ? std::min(a: ThreadLimit, b: UB) : UB;
8859 if (!llvm::to_integer(S: LBStr, Num&: LB, Base: 10))
8860 return {0, UB};
8861 return {LB, UB};
8862 }
8863
8864 if (Kernel.hasFnAttribute(Kind: NVVMAttr::MaxNTID)) {
8865 int32_t UB = Kernel.getFnAttributeAsParsedInteger(Kind: NVVMAttr::MaxNTID);
8866 return {0, ThreadLimit ? std::min(a: ThreadLimit, b: UB) : UB};
8867 }
8868 return {0, ThreadLimit};
8869}
8870
8871void OpenMPIRBuilder::writeThreadBoundsForKernel(const Triple &T,
8872 Function &Kernel, int32_t LB,
8873 int32_t UB) {
8874 Kernel.addFnAttr(Kind: "omp_target_thread_limit", Val: std::to_string(val: UB));
8875
8876 if (T.isAMDGPU()) {
8877 Kernel.addFnAttr(Kind: "amdgpu-flat-work-group-size",
8878 Val: llvm::utostr(X: LB) + "," + llvm::utostr(X: UB));
8879 return;
8880 }
8881
8882 updateNVPTXAttr(Kernel, Name: NVVMAttr::MaxNTID, Value: UB, Min: true);
8883}
8884
8885std::pair<int32_t, int32_t>
8886OpenMPIRBuilder::readTeamBoundsForKernel(const Triple &, Function &Kernel) {
8887 // TODO: Read from backend annotations if available.
8888 return {0, Kernel.getFnAttributeAsParsedInteger(Kind: "omp_target_num_teams")};
8889}
8890
8891void OpenMPIRBuilder::writeTeamsForKernel(const Triple &T, Function &Kernel,
8892 int32_t LB, int32_t UB) {
8893 if (UB > 0) {
8894 if (T.isNVPTX())
8895 Kernel.addFnAttr(Kind: NVVMAttr::MaxClusterRank, Val: llvm::utostr(X: UB));
8896 if (T.isAMDGPU())
8897 Kernel.addFnAttr(Kind: "amdgpu-max-num-workgroups", Val: llvm::utostr(X: UB) + ",1,1");
8898 }
8899
8900 Kernel.addFnAttr(Kind: "omp_target_num_teams", Val: std::to_string(val: LB));
8901}
8902
8903void OpenMPIRBuilder::setOutlinedTargetRegionFunctionAttributes(
8904 Function *OutlinedFn) {
8905 if (Config.isTargetDevice()) {
8906 OutlinedFn->setLinkage(GlobalValue::WeakODRLinkage);
8907 // TODO: Determine if DSO local can be set to true.
8908 OutlinedFn->setDSOLocal(false);
8909 OutlinedFn->setVisibility(GlobalValue::ProtectedVisibility);
8910 if (T.isAMDGCN())
8911 OutlinedFn->setCallingConv(CallingConv::AMDGPU_KERNEL);
8912 else if (T.isNVPTX())
8913 OutlinedFn->setCallingConv(CallingConv::PTX_Kernel);
8914 else if (T.isSPIRV())
8915 OutlinedFn->setCallingConv(CallingConv::SPIR_KERNEL);
8916 }
8917}
8918
8919Constant *OpenMPIRBuilder::createOutlinedFunctionID(Function *OutlinedFn,
8920 StringRef EntryFnIDName) {
8921 if (Config.isTargetDevice()) {
8922 assert(OutlinedFn && "The outlined function must exist if embedded");
8923 return OutlinedFn;
8924 }
8925
8926 return new GlobalVariable(
8927 M, Builder.getInt8Ty(), /*isConstant=*/true, GlobalValue::WeakAnyLinkage,
8928 Constant::getNullValue(Ty: Builder.getInt8Ty()), EntryFnIDName);
8929}
8930
8931Constant *OpenMPIRBuilder::createTargetRegionEntryAddr(Function *OutlinedFn,
8932 StringRef EntryFnName) {
8933 if (OutlinedFn)
8934 return OutlinedFn;
8935
8936 assert(!M.getGlobalVariable(EntryFnName, true) &&
8937 "Named kernel already exists?");
8938 return new GlobalVariable(
8939 M, Builder.getInt8Ty(), /*isConstant=*/true, GlobalValue::InternalLinkage,
8940 Constant::getNullValue(Ty: Builder.getInt8Ty()), EntryFnName);
8941}
8942
8943Error OpenMPIRBuilder::emitTargetRegionFunction(
8944 TargetRegionEntryInfo &EntryInfo,
8945 FunctionGenCallback &GenerateFunctionCallback, bool IsOffloadEntry,
8946 Function *&OutlinedFn, Constant *&OutlinedFnID) {
8947
8948 SmallString<64> EntryFnName;
8949 OffloadInfoManager.getTargetRegionEntryFnName(Name&: EntryFnName, EntryInfo);
8950
8951 if (Config.isTargetDevice() || !Config.openMPOffloadMandatory()) {
8952 Expected<Function *> CBResult = GenerateFunctionCallback(EntryFnName);
8953 if (!CBResult)
8954 return CBResult.takeError();
8955 OutlinedFn = *CBResult;
8956 } else {
8957 OutlinedFn = nullptr;
8958 }
8959
8960 // If this target outline function is not an offload entry, we don't need to
8961 // register it. This may be in the case of a false if clause, or if there are
8962 // no OpenMP targets.
8963 if (!IsOffloadEntry)
8964 return Error::success();
8965
8966 std::string EntryFnIDName =
8967 Config.isTargetDevice()
8968 ? std::string(EntryFnName)
8969 : createPlatformSpecificName(Parts: {EntryFnName, "region_id"});
8970
8971 OutlinedFnID = registerTargetRegionFunction(EntryInfo, OutlinedFunction: OutlinedFn,
8972 EntryFnName, EntryFnIDName);
8973 return Error::success();
8974}
8975
8976Constant *OpenMPIRBuilder::registerTargetRegionFunction(
8977 TargetRegionEntryInfo &EntryInfo, Function *OutlinedFn,
8978 StringRef EntryFnName, StringRef EntryFnIDName) {
8979 if (OutlinedFn)
8980 setOutlinedTargetRegionFunctionAttributes(OutlinedFn);
8981 auto OutlinedFnID = createOutlinedFunctionID(OutlinedFn, EntryFnIDName);
8982 auto EntryAddr = createTargetRegionEntryAddr(OutlinedFn, EntryFnName);
8983 OffloadInfoManager.registerTargetRegionEntryInfo(
8984 EntryInfo, Addr: EntryAddr, ID: OutlinedFnID,
8985 Flags: OffloadEntriesInfoManager::OMPTargetRegionEntryTargetRegion);
8986 return OutlinedFnID;
8987}
8988
8989OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTargetData(
8990 const LocationDescription &Loc, InsertPointTy AllocaIP,
8991 InsertPointTy CodeGenIP, ArrayRef<BasicBlock *> DeallocBlocks,
8992 Value *DeviceID, Value *IfCond, TargetDataInfo &Info,
8993 GenMapInfoCallbackTy GenMapInfoCB, CustomMapperCallbackTy CustomMapperCB,
8994 omp::RuntimeFunction *MapperFunc,
8995 function_ref<InsertPointOrErrorTy(InsertPointTy CodeGenIP,
8996 BodyGenTy BodyGenType)>
8997 BodyGenCB,
8998 function_ref<void(unsigned int, Value *)> DeviceAddrCB, Value *SrcLocInfo) {
8999 if (!updateToLocation(Loc))
9000 return InsertPointTy();
9001
9002 Builder.restoreIP(IP: CodeGenIP);
9003
9004 bool IsStandAlone = !BodyGenCB;
9005 MapInfosTy *MapInfo;
9006 // Generate the code for the opening of the data environment. Capture all the
9007 // arguments of the runtime call by reference because they are used in the
9008 // closing of the region.
9009 auto BeginThenGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
9010 ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
9011 MapInfo = &GenMapInfoCB(Builder.saveIP());
9012 if (Error Err = emitOffloadingArrays(
9013 AllocaIP, CodeGenIP: Builder.saveIP(), CombinedInfo&: *MapInfo, Info, CustomMapperCB,
9014 /*IsNonContiguous=*/true, DeviceAddrCB))
9015 return Err;
9016
9017 TargetDataRTArgs RTArgs;
9018 emitOffloadingArraysArgument(Builder, RTArgs, Info);
9019
9020 // Emit the number of elements in the offloading arrays.
9021 Value *PointerNum = Builder.getInt32(C: Info.NumberOfPtrs);
9022
9023 // Source location for the ident struct
9024 if (!SrcLocInfo) {
9025 uint32_t SrcLocStrSize;
9026 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
9027 SrcLocInfo = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
9028 }
9029
9030 SmallVector<llvm::Value *, 13> OffloadingArgs = {
9031 SrcLocInfo, DeviceID,
9032 PointerNum, RTArgs.BasePointersArray,
9033 RTArgs.PointersArray, RTArgs.SizesArray,
9034 RTArgs.MapTypesArray, RTArgs.MapNamesArray,
9035 RTArgs.MappersArray};
9036
9037 if (IsStandAlone) {
9038 assert(MapperFunc && "MapperFunc missing for standalone target data");
9039
9040 auto TaskBodyCB = [&](Value *, Value *,
9041 IRBuilderBase::InsertPoint) -> Error {
9042 if (Info.HasNoWait) {
9043 OffloadingArgs.append(IL: {llvm::Constant::getNullValue(Ty: Int32),
9044 llvm::Constant::getNullValue(Ty: VoidPtr),
9045 llvm::Constant::getNullValue(Ty: Int32),
9046 llvm::Constant::getNullValue(Ty: VoidPtr)});
9047 }
9048
9049 createRuntimeFunctionCall(Callee: getOrCreateRuntimeFunctionPtr(FnID: *MapperFunc),
9050 Args: OffloadingArgs);
9051
9052 if (Info.HasNoWait) {
9053 BasicBlock *OffloadContBlock =
9054 BasicBlock::Create(Context&: Builder.getContext(), Name: "omp_offload.cont");
9055 Function *CurFn = Builder.GetInsertBlock()->getParent();
9056 emitBlock(BB: OffloadContBlock, CurFn, /*IsFinished=*/true);
9057 Builder.restoreIP(IP: Builder.saveIP());
9058 }
9059 return Error::success();
9060 };
9061
9062 bool RequiresOuterTargetTask = Info.HasNoWait;
9063 if (!RequiresOuterTargetTask)
9064 cantFail(Err: TaskBodyCB(/*DeviceID=*/nullptr, /*RTLoc=*/nullptr,
9065 /*TargetTaskAllocaIP=*/{}));
9066 else
9067 cantFail(ValOrErr: emitTargetTask(TaskBodyCB, DeviceID, RTLoc: SrcLocInfo, AllocaIP,
9068 /*Dependencies=*/{}, RTArgs, HasNoWait: Info.HasNoWait));
9069 } else {
9070 Function *BeginMapperFunc = getOrCreateRuntimeFunctionPtr(
9071 FnID: omp::OMPRTL___tgt_target_data_begin_mapper);
9072
9073 createRuntimeFunctionCall(Callee: BeginMapperFunc, Args: OffloadingArgs);
9074
9075 for (auto DeviceMap : Info.DevicePtrInfoMap) {
9076 if (isa<AllocaInst>(Val: DeviceMap.second.second)) {
9077 auto *LI =
9078 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: DeviceMap.second.first);
9079 Builder.CreateStore(Val: LI, Ptr: DeviceMap.second.second);
9080 }
9081 }
9082
9083 // If device pointer privatization is required, emit the body of the
9084 // region here. It will have to be duplicated: with and without
9085 // privatization.
9086 InsertPointOrErrorTy AfterIP =
9087 BodyGenCB(Builder.saveIP(), BodyGenTy::Priv);
9088 if (!AfterIP)
9089 return AfterIP.takeError();
9090 Builder.restoreIP(IP: *AfterIP);
9091 }
9092 return Error::success();
9093 };
9094
9095 // If we need device pointer privatization, we need to emit the body of the
9096 // region with no privatization in the 'else' branch of the conditional.
9097 // Otherwise, we don't have to do anything.
9098 auto BeginElseGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
9099 ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
9100 InsertPointOrErrorTy AfterIP =
9101 BodyGenCB(Builder.saveIP(), BodyGenTy::DupNoPriv);
9102 if (!AfterIP)
9103 return AfterIP.takeError();
9104 Builder.restoreIP(IP: *AfterIP);
9105 return Error::success();
9106 };
9107
9108 // Generate code for the closing of the data region.
9109 auto EndThenGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
9110 ArrayRef<BasicBlock *> DeallocBlocks) {
9111 TargetDataRTArgs RTArgs;
9112 Info.EmitDebug = !MapInfo->Names.empty();
9113 emitOffloadingArraysArgument(Builder, RTArgs, Info, /*ForEndCall=*/true);
9114
9115 // Emit the number of elements in the offloading arrays.
9116 Value *PointerNum = Builder.getInt32(C: Info.NumberOfPtrs);
9117
9118 // Source location for the ident struct
9119 if (!SrcLocInfo) {
9120 uint32_t SrcLocStrSize;
9121 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
9122 SrcLocInfo = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
9123 }
9124
9125 Value *OffloadingArgs[] = {SrcLocInfo, DeviceID,
9126 PointerNum, RTArgs.BasePointersArray,
9127 RTArgs.PointersArray, RTArgs.SizesArray,
9128 RTArgs.MapTypesArray, RTArgs.MapNamesArray,
9129 RTArgs.MappersArray};
9130 Function *EndMapperFunc =
9131 getOrCreateRuntimeFunctionPtr(FnID: omp::OMPRTL___tgt_target_data_end_mapper);
9132
9133 createRuntimeFunctionCall(Callee: EndMapperFunc, Args: OffloadingArgs);
9134 return Error::success();
9135 };
9136
9137 // We don't have to do anything to close the region if the if clause evaluates
9138 // to false.
9139 auto EndElseGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
9140 ArrayRef<BasicBlock *> DeallocBlocks) {
9141 return Error::success();
9142 };
9143
9144 Error Err = [&]() -> Error {
9145 if (BodyGenCB) {
9146 Error Err = [&]() {
9147 if (IfCond)
9148 return emitIfClause(Cond: IfCond, ThenGen: BeginThenGen, ElseGen: BeginElseGen, AllocaIP);
9149 return BeginThenGen(AllocaIP, Builder.saveIP(), DeallocBlocks);
9150 }();
9151
9152 if (Err)
9153 return Err;
9154
9155 // If we don't require privatization of device pointers, we emit the body
9156 // in between the runtime calls. This avoids duplicating the body code.
9157 InsertPointOrErrorTy AfterIP =
9158 BodyGenCB(Builder.saveIP(), BodyGenTy::NoPriv);
9159 if (!AfterIP)
9160 return AfterIP.takeError();
9161 restoreIPandDebugLoc(Builder, IP: *AfterIP);
9162
9163 if (IfCond)
9164 return emitIfClause(Cond: IfCond, ThenGen: EndThenGen, ElseGen: EndElseGen, AllocaIP);
9165 return EndThenGen(AllocaIP, Builder.saveIP(), DeallocBlocks);
9166 }
9167 if (IfCond)
9168 return emitIfClause(Cond: IfCond, ThenGen: BeginThenGen, ElseGen: EndElseGen, AllocaIP);
9169 return BeginThenGen(AllocaIP, Builder.saveIP(), DeallocBlocks);
9170 }();
9171
9172 if (Err)
9173 return Err;
9174
9175 return Builder.saveIP();
9176}
9177
9178FunctionCallee
9179OpenMPIRBuilder::createForStaticInitFunction(unsigned IVSize, bool IVSigned,
9180 bool IsGPUDistribute) {
9181 assert((IVSize == 32 || IVSize == 64) &&
9182 "IV size is not compatible with the omp runtime");
9183 RuntimeFunction Name;
9184 if (IsGPUDistribute)
9185 Name = IVSize == 32
9186 ? (IVSigned ? omp::OMPRTL___kmpc_distribute_static_init_4
9187 : omp::OMPRTL___kmpc_distribute_static_init_4u)
9188 : (IVSigned ? omp::OMPRTL___kmpc_distribute_static_init_8
9189 : omp::OMPRTL___kmpc_distribute_static_init_8u);
9190 else
9191 Name = IVSize == 32 ? (IVSigned ? omp::OMPRTL___kmpc_for_static_init_4
9192 : omp::OMPRTL___kmpc_for_static_init_4u)
9193 : (IVSigned ? omp::OMPRTL___kmpc_for_static_init_8
9194 : omp::OMPRTL___kmpc_for_static_init_8u);
9195
9196 return getOrCreateRuntimeFunction(M, FnID: Name);
9197}
9198
9199FunctionCallee OpenMPIRBuilder::createDispatchInitFunction(unsigned IVSize,
9200 bool IVSigned) {
9201 assert((IVSize == 32 || IVSize == 64) &&
9202 "IV size is not compatible with the omp runtime");
9203 RuntimeFunction Name = IVSize == 32
9204 ? (IVSigned ? omp::OMPRTL___kmpc_dispatch_init_4
9205 : omp::OMPRTL___kmpc_dispatch_init_4u)
9206 : (IVSigned ? omp::OMPRTL___kmpc_dispatch_init_8
9207 : omp::OMPRTL___kmpc_dispatch_init_8u);
9208
9209 return getOrCreateRuntimeFunction(M, FnID: Name);
9210}
9211
9212FunctionCallee OpenMPIRBuilder::createDispatchNextFunction(unsigned IVSize,
9213 bool IVSigned) {
9214 assert((IVSize == 32 || IVSize == 64) &&
9215 "IV size is not compatible with the omp runtime");
9216 RuntimeFunction Name = IVSize == 32
9217 ? (IVSigned ? omp::OMPRTL___kmpc_dispatch_next_4
9218 : omp::OMPRTL___kmpc_dispatch_next_4u)
9219 : (IVSigned ? omp::OMPRTL___kmpc_dispatch_next_8
9220 : omp::OMPRTL___kmpc_dispatch_next_8u);
9221
9222 return getOrCreateRuntimeFunction(M, FnID: Name);
9223}
9224
9225FunctionCallee OpenMPIRBuilder::createDispatchFiniFunction(unsigned IVSize,
9226 bool IVSigned) {
9227 assert((IVSize == 32 || IVSize == 64) &&
9228 "IV size is not compatible with the omp runtime");
9229 RuntimeFunction Name = IVSize == 32
9230 ? (IVSigned ? omp::OMPRTL___kmpc_dispatch_fini_4
9231 : omp::OMPRTL___kmpc_dispatch_fini_4u)
9232 : (IVSigned ? omp::OMPRTL___kmpc_dispatch_fini_8
9233 : omp::OMPRTL___kmpc_dispatch_fini_8u);
9234
9235 return getOrCreateRuntimeFunction(M, FnID: Name);
9236}
9237
9238FunctionCallee OpenMPIRBuilder::createDispatchDeinitFunction() {
9239 return getOrCreateRuntimeFunction(M, FnID: omp::OMPRTL___kmpc_dispatch_deinit);
9240}
9241
9242static void FixupDebugInfoForOutlinedFunction(
9243 OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, Function *Func,
9244 DenseMap<Value *, std::tuple<Value *, unsigned>> &ValueReplacementMap) {
9245
9246 DISubprogram *NewSP = Func->getSubprogram();
9247 if (!NewSP)
9248 return;
9249
9250 SmallDenseMap<DILocalVariable *, DILocalVariable *> RemappedVariables;
9251
9252 auto GetUpdatedDIVariable = [&](DILocalVariable *OldVar, unsigned arg) {
9253 DILocalVariable *&NewVar = RemappedVariables[OldVar];
9254 // Only use cached variable if the arg number matches. This is important
9255 // so that DIVariable created for privatized variables are not discarded.
9256 if (NewVar && (arg == NewVar->getArg()))
9257 return NewVar;
9258
9259 NewVar = llvm::DILocalVariable::get(
9260 Context&: Builder.getContext(), Scope: OldVar->getScope(), Name: OldVar->getName(),
9261 File: OldVar->getFile(), Line: OldVar->getLine(), Type: OldVar->getType(), Arg: arg,
9262 Flags: OldVar->getFlags(), AlignInBits: OldVar->getAlignInBits(), Annotations: OldVar->getAnnotations());
9263 return NewVar;
9264 };
9265
9266 auto UpdateDebugRecord = [&](auto *DR) {
9267 DILocalVariable *OldVar = DR->getVariable();
9268 unsigned ArgNo = 0;
9269 for (auto Loc : DR->location_ops()) {
9270 auto Iter = ValueReplacementMap.find(Loc);
9271 if (Iter != ValueReplacementMap.end()) {
9272 DR->replaceVariableLocationOp(Loc, std::get<0>(Iter->second));
9273 ArgNo = std::get<1>(Iter->second) + 1;
9274 }
9275 }
9276 if (ArgNo != 0)
9277 DR->setVariable(GetUpdatedDIVariable(OldVar, ArgNo));
9278 };
9279
9280 SmallVector<DbgVariableRecord *, 4> DVRsToDelete;
9281 auto MoveDebugRecordToCorrectBlock = [&](DbgVariableRecord *DVR) {
9282 if (DVR->getNumVariableLocationOps() != 1u) {
9283 DVR->setKillLocation();
9284 return;
9285 }
9286 Value *Loc = DVR->getVariableLocationOp(OpIdx: 0u);
9287 BasicBlock *CurBB = DVR->getParent();
9288 BasicBlock *RequiredBB = nullptr;
9289
9290 if (Instruction *LocInst = dyn_cast<Instruction>(Val: Loc))
9291 RequiredBB = LocInst->getParent();
9292 else if (isa<llvm::Argument>(Val: Loc))
9293 RequiredBB = &DVR->getFunction()->getEntryBlock();
9294
9295 if (RequiredBB && RequiredBB != CurBB) {
9296 assert(!RequiredBB->empty());
9297 RequiredBB->insertDbgRecordBefore(DR: DVR->clone(),
9298 Here: RequiredBB->back().getIterator());
9299 DVRsToDelete.push_back(Elt: DVR);
9300 }
9301 };
9302
9303 // The location and scope of variable records still point to
9304 // the parent function of the target region. Update them.
9305 for (Instruction &I : instructions(F: Func)) {
9306 for (DbgVariableRecord &DVR : filterDbgVars(R: I.getDbgRecordRange())) {
9307 UpdateDebugRecord(&DVR);
9308 MoveDebugRecordToCorrectBlock(&DVR);
9309 }
9310 }
9311 for (auto *DVR : DVRsToDelete)
9312 DVR->getMarker()->MarkedInstr->dropOneDbgRecord(I: DVR);
9313 // An extra argument is passed to the device. Create the debug data for it.
9314 if (OMPBuilder.Config.isTargetDevice()) {
9315 DICompileUnit *CU = NewSP->getUnit();
9316 Module *M = Func->getParent();
9317 DIBuilder DB(*M, true, CU);
9318 DIType *VoidPtrTy =
9319 DB.createQualifiedType(Tag: dwarf::DW_TAG_pointer_type, FromTy: nullptr);
9320 unsigned ArgNo = Func->arg_size();
9321 DILocalVariable *Var = DB.createParameterVariable(
9322 Scope: NewSP, Name: "dyn_ptr", ArgNo, File: NewSP->getFile(), /*LineNo=*/0, Ty: VoidPtrTy,
9323 /*AlwaysPreserve=*/false, Flags: DINode::DIFlags::FlagArtificial);
9324 auto Loc = DILocation::get(Context&: Func->getContext(), Line: 0, Column: 0, Scope: NewSP, InlinedAt: 0);
9325 Argument *LastArg = Func->getArg(i: Func->arg_size() - 1);
9326 DB.insertDeclare(Storage: LastArg, VarInfo: Var, Expr: DB.createExpression(), DL: Loc,
9327 InsertAtEnd: &(*Func->begin()));
9328 }
9329}
9330
9331static Value *removeASCastIfPresent(Value *V) {
9332 if (Operator::getOpcode(V) == Instruction::AddrSpaceCast)
9333 return cast<Operator>(Val: V)->getOperand(i: 0);
9334 return V;
9335}
9336
9337static Expected<Function *> createOutlinedFunction(
9338 OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder,
9339 const OpenMPIRBuilder::TargetKernelDefaultAttrs &DefaultAttrs,
9340 StringRef FuncName, SmallVectorImpl<Value *> &Inputs,
9341 OpenMPIRBuilder::TargetBodyGenCallbackTy &CBFunc,
9342 OpenMPIRBuilder::TargetGenArgAccessorsCallbackTy &ArgAccessorFuncCB,
9343 DebugLoc OutlinedFnLoc) {
9344 SmallVector<Type *> ParameterTypes;
9345 if (OMPBuilder.Config.isTargetDevice()) {
9346 // All parameters to target devices are passed as pointers
9347 // or i64. This assumes 64-bit address spaces/pointers.
9348 for (auto &Arg : Inputs)
9349 ParameterTypes.push_back(Elt: Arg->getType()->isPointerTy()
9350 ? Arg->getType()
9351 : Type::getInt64Ty(C&: Builder.getContext()));
9352 } else {
9353 for (auto &Arg : Inputs)
9354 ParameterTypes.push_back(Elt: Arg->getType());
9355 }
9356
9357 // The implicit dyn_ptr argument is always the last parameter on both host
9358 // and device so the argument counts match without runtime manipulation.
9359 auto *PtrTy = PointerType::getUnqual(C&: Builder.getContext());
9360 ParameterTypes.push_back(Elt: PtrTy);
9361
9362 auto BB = Builder.GetInsertBlock();
9363 auto M = BB->getModule();
9364 auto FuncType = FunctionType::get(Result: Builder.getVoidTy(), Params: ParameterTypes,
9365 /*isVarArg*/ false);
9366 auto Func =
9367 Function::Create(Ty: FuncType, Linkage: GlobalValue::InternalLinkage, N: FuncName, M);
9368
9369 // Forward target-cpu and target-features function attributes from the
9370 // original function to the new outlined function.
9371 Function *ParentFn = Builder.GetInsertBlock()->getParent();
9372
9373 auto TargetCpuAttr = ParentFn->getFnAttribute(Kind: "target-cpu");
9374 if (TargetCpuAttr.isStringAttribute())
9375 Func->addFnAttr(Attr: TargetCpuAttr);
9376
9377 auto TargetFeaturesAttr = ParentFn->getFnAttribute(Kind: "target-features");
9378 if (TargetFeaturesAttr.isStringAttribute())
9379 Func->addFnAttr(Attr: TargetFeaturesAttr);
9380
9381 if (OMPBuilder.Config.isTargetDevice()) {
9382 Value *ExecMode =
9383 OMPBuilder.emitKernelExecutionMode(KernelName: FuncName, Mode: DefaultAttrs.ExecFlags);
9384 OMPBuilder.emitUsed(Name: "llvm.compiler.used", List: {ExecMode});
9385 }
9386
9387 // Save insert point.
9388 IRBuilder<>::InsertPointGuard IPG(Builder);
9389 // We will generate the entries in the outlined function but the debug
9390 // location is still pointing to the parent function, which is the wrong
9391 // scope. OutlinedFnLoc, when the caller provides one, is the same source
9392 // position scoped to the subprogram that will be attached to the outlined
9393 // function, so it is what everything emitted below needs.
9394 Builder.SetCurrentDebugLocation(OutlinedFnLoc);
9395
9396 // Generate the region into the function.
9397 BasicBlock *EntryBB = BasicBlock::Create(Context&: Builder.getContext(), Name: "entry", Parent: Func);
9398 Builder.SetInsertPoint(EntryBB);
9399
9400 // Insert target init call in the device compilation pass. On the host
9401 // (e.g. a non-GPU offload target), there is no runtime init/deinit
9402 // sequence, but the runtime still needs a '<kernel>_kernel_environment'
9403 // global to know how the kernel was configured, so emit it directly.
9404 if (OMPBuilder.Config.isTargetDevice())
9405 Builder.restoreIP(IP: OMPBuilder.createTargetInit(Loc: Builder, Attrs: DefaultAttrs));
9406 else
9407 OMPBuilder.emitKernelEnvironment(Loc: Builder, Attrs: DefaultAttrs);
9408
9409 BasicBlock *UserCodeEntryBB = Builder.GetInsertBlock();
9410
9411 // As we embed the user code in the middle of our target region after we
9412 // generate entry code, we must move what allocas we can into the entry
9413 // block to avoid possible breaking optimisations
9414 OMPBuilder.ConstantAllocaRaiseCandidates.emplace_back(Args&: Func);
9415
9416 BasicBlock *ExitBB = splitBB(Builder, /*CreateBranch=*/true, Name: "target.exit");
9417 BasicBlock *OutlinedBodyBB =
9418 splitBB(Builder, /*CreateBranch=*/true, Name: "outlined.body");
9419 llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
9420 CBFunc(Builder.saveIP(), OutlinedBodyBB->begin(), ExitBB);
9421 if (!AfterIP)
9422 return AfterIP.takeError();
9423 Builder.SetInsertPoint(ExitBB);
9424 // The body callback builds the body with its own IRBuilder and cannot reach
9425 // this one directly. But a body holding another OpenMP construct, a nested
9426 // parallel say, calls OpenMPIRBuilder::createParallel, and that can leave
9427 // this Builder pointing at the wrong debug location, or at none at all. The
9428 // epilogue below belongs to the target construct rather than to whatever the
9429 // body emitted last, so re-establish the location the prologue was emitted
9430 // with.
9431 Builder.SetCurrentDebugLocation(OutlinedFnLoc);
9432
9433 // Insert target deinit call in the device compilation pass.
9434 if (OMPBuilder.Config.isTargetDevice())
9435 OMPBuilder.createTargetDeinit(Loc: Builder);
9436
9437 // Insert return instruction.
9438 Builder.CreateRetVoid();
9439
9440 // New Alloca IP at entry point of created device function.
9441 Builder.SetInsertPoint(EntryBB->getFirstNonPHIIt());
9442 auto AllocaIP = Builder.saveIP();
9443
9444 Builder.SetInsertPoint(UserCodeEntryBB->getFirstNonPHIOrPseudoProbe());
9445
9446 // Do not include the artificial dyn_ptr argument.
9447 const auto &ArgRange = make_range(x: Func->arg_begin(), y: Func->arg_end() - 1);
9448
9449 DenseMap<Value *, std::tuple<Value *, unsigned>> ValueReplacementMap;
9450
9451 auto ReplaceValue = [](Value *Input, Value *InputCopy, Function *Func) {
9452 // Things like GEP's can come in the form of Constants. Constants and
9453 // ConstantExpr's do not have access to the knowledge of what they're
9454 // contained in, so we must dig a little to find an instruction so we
9455 // can tell if they're used inside of the function we're outlining. We
9456 // also replace the original constant expression with a new instruction
9457 // equivalent; an instruction as it allows easy modification in the
9458 // following loop, as we can now know the constant (instruction) is
9459 // owned by our target function and replaceUsesOfWith can now be invoked
9460 // on it (cannot do this with constants it seems). A brand new one also
9461 // allows us to be cautious as it is perhaps possible the old expression
9462 // was used inside of the function but exists and is used externally
9463 // (unlikely by the nature of a Constant, but still).
9464 // NOTE: We cannot remove dead constants that have been rewritten to
9465 // instructions at this stage, we run the risk of breaking later lowering
9466 // by doing so as we could still be in the process of lowering the module
9467 // from MLIR to LLVM-IR and the MLIR lowering may still require the original
9468 // constants we have created rewritten versions of.
9469 if (auto *Const = dyn_cast<Constant>(Val: Input))
9470 convertUsersOfConstantsToInstructions(Consts: Const, RestrictToFunc: Func, RemoveDeadConstants: false);
9471
9472 // Collect users before iterating over them to avoid invalidating the
9473 // iteration in case a user uses Input more than once (e.g. a call
9474 // instruction).
9475 SetVector<User *> Users(Input->users().begin(), Input->users().end());
9476 // Collect all the instructions
9477 for (User *User : make_early_inc_range(Range&: Users))
9478 if (auto *Instr = dyn_cast<Instruction>(Val: User))
9479 if (Instr->getFunction() == Func)
9480 Instr->replaceUsesOfWith(From: Input, To: InputCopy);
9481 };
9482
9483 SmallVector<std::pair<Value *, Value *>> DeferredReplacement;
9484
9485 // Rewrite uses of input valus to parameters.
9486 for (auto InArg : zip(t&: Inputs, u: ArgRange)) {
9487 Value *Input = std::get<0>(t&: InArg);
9488 Argument &Arg = std::get<1>(t&: InArg);
9489 Value *InputCopy = nullptr;
9490
9491 llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = ArgAccessorFuncCB(
9492 Arg, Input, InputCopy, AllocaIP, Builder.saveIP(), ExitBB->begin());
9493 if (!AfterIP)
9494 return AfterIP.takeError();
9495 Builder.restoreIP(IP: *AfterIP);
9496 ValueReplacementMap[Input] = std::make_tuple(args&: InputCopy, args: Arg.getArgNo());
9497
9498 // In certain cases a Global may be set up for replacement, however, this
9499 // Global may be used in multiple arguments to the kernel, just segmented
9500 // apart, for example, if we have a global array, that is sectioned into
9501 // multiple mappings (technically not legal in OpenMP, but there is a case
9502 // in Fortran for Common Blocks where this is neccesary), we will end up
9503 // with GEP's into this array inside the kernel, that refer to the Global
9504 // but are technically separate arguments to the kernel for all intents and
9505 // purposes. If we have mapped a segment that requires a GEP into the 0-th
9506 // index, it will fold into an referal to the Global, if we then encounter
9507 // this folded GEP during replacement all of the references to the
9508 // Global in the kernel will be replaced with the argument we have generated
9509 // that corresponds to it, including any other GEP's that refer to the
9510 // Global that may be other arguments. This will invalidate all of the other
9511 // preceding mapped arguments that refer to the same global that may be
9512 // separate segments. To prevent this, we defer global processing until all
9513 // other processing has been performed.
9514 if (llvm::isa<llvm::GlobalValue, llvm::GlobalObject, llvm::GlobalVariable>(
9515 Val: removeASCastIfPresent(V: Input))) {
9516 DeferredReplacement.push_back(Elt: std::make_pair(x&: Input, y&: InputCopy));
9517 continue;
9518 }
9519
9520 if (isa<ConstantData>(Val: Input))
9521 continue;
9522
9523 ReplaceValue(Input, InputCopy, Func);
9524 }
9525
9526 // Replace all of our deferred Input values, currently just Globals.
9527 for (auto Deferred : DeferredReplacement)
9528 ReplaceValue(std::get<0>(in&: Deferred), std::get<1>(in&: Deferred), Func);
9529
9530 FixupDebugInfoForOutlinedFunction(OMPBuilder, Builder, Func,
9531 ValueReplacementMap);
9532 return Func;
9533}
9534/// Given a task descriptor, TaskWithPrivates, return the pointer to the block
9535/// of pointers containing shared data between the parent task and the created
9536/// task.
9537static LoadInst *loadSharedDataFromTaskDescriptor(IRBuilderBase &Builder,
9538 Value *TaskWithPrivates) {
9539 LLVMContext &Ctx = Builder.getContext();
9540 return Builder.CreateLoad(Ty: PointerType::getUnqual(C&: Ctx), Ptr: TaskWithPrivates);
9541}
9542/// Create an entry point for a target task with the following.
9543/// It'll have the following signature
9544/// void @.omp_target_task_proxy_func(i32 %thread.id, ptr %task)
9545/// This function is called from emitTargetTask once the
9546/// code to launch the target kernel has been outlined already.
9547/// NumOffloadingArrays is the number of offloading arrays that we need to copy
9548/// into the task structure so that the deferred target task can access this
9549/// data even after the stack frame of the generating task has been rolled
9550/// back. Offloading arrays contain base pointers, pointers, sizes etc
9551/// of the data that the target kernel will access. These in effect are the
9552/// non-empty arrays of pointers held by OpenMPIRBuilder::TargetDataRTArgs.
9553static Function *emitTargetTaskProxyFunction(
9554 OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, CallInst *StaleCI,
9555 StructType *PrivatesTy, StructType *TaskWithPrivatesTy,
9556 const size_t NumOffloadingArrays, const int SharedArgsOperandNo) {
9557
9558 // If NumOffloadingArrays is non-zero, PrivatesTy better not be nullptr.
9559 // This is because PrivatesTy is the type of the structure in which
9560 // we pass the offloading arrays to the deferred target task.
9561 assert((!NumOffloadingArrays || PrivatesTy) &&
9562 "PrivatesTy cannot be nullptr when there are offloadingArrays"
9563 "to privatize");
9564
9565 Module &M = OMPBuilder.M;
9566 // KernelLaunchFunction is the target launch function, i.e.
9567 // the function that sets up kernel arguments and calls
9568 // __tgt_target_kernel to launch the kernel on the device.
9569 //
9570 Function *KernelLaunchFunction = StaleCI->getCalledFunction();
9571
9572 // StaleCI is the CallInst which is the call to the outlined
9573 // target kernel launch function. If there are local live-in values
9574 // that the outlined function uses then these are aggregated into a structure
9575 // which is passed as the second argument. If there are no local live-in
9576 // values or if all values used by the outlined kernel are global variables,
9577 // then there's only one argument, the threadID. So, StaleCI can be
9578 //
9579 // %structArg = alloca { ptr, ptr }, align 8
9580 // %gep_ = getelementptr { ptr, ptr }, ptr %structArg, i32 0, i32 0
9581 // store ptr %20, ptr %gep_, align 8
9582 // %gep_8 = getelementptr { ptr, ptr }, ptr %structArg, i32 0, i32 1
9583 // store ptr %21, ptr %gep_8, align 8
9584 // call void @_QQmain..omp_par.1(i32 %global.tid.val6, ptr %structArg)
9585 //
9586 // OR
9587 //
9588 // call void @_QQmain..omp_par.1(i32 %global.tid.val6)
9589 LLVMContext &Ctx = StaleCI->getParent()->getContext();
9590
9591 Type *ThreadIDTy = Type::getInt32Ty(C&: Ctx);
9592 Type *TaskPtrTy = OMPBuilder.TaskPtr;
9593 [[maybe_unused]] Type *TaskTy = OMPBuilder.Task;
9594
9595 auto ProxyFnTy =
9596 FunctionType::get(Result: Builder.getVoidTy(), Params: {ThreadIDTy, TaskPtrTy},
9597 /* isVarArg */ false);
9598 auto ProxyFn = Function::Create(Ty: ProxyFnTy, Linkage: GlobalValue::InternalLinkage,
9599 N: ".omp_target_task_proxy_func", M);
9600 Value *ThreadId = ProxyFn->getArg(i: 0);
9601 Value *TaskWithPrivates = ProxyFn->getArg(i: 1);
9602 ThreadId->setName("thread.id");
9603 TaskWithPrivates->setName("task");
9604
9605 bool HasShareds = SharedArgsOperandNo > 0;
9606 bool HasOffloadingArrays = NumOffloadingArrays > 0;
9607 IRBuilder<>::InsertPointGuard IPG(Builder);
9608 BasicBlock *EntryBB =
9609 BasicBlock::Create(Context&: Builder.getContext(), Name: "entry", Parent: ProxyFn);
9610 Builder.SetInsertPoint(EntryBB);
9611 Builder.SetCurrentDebugLocation(llvm::DebugLoc());
9612
9613 SmallVector<Value *> KernelLaunchArgs;
9614 KernelLaunchArgs.reserve(N: StaleCI->arg_size());
9615 KernelLaunchArgs.push_back(Elt: ThreadId);
9616
9617 if (HasOffloadingArrays) {
9618 assert(TaskTy != TaskWithPrivatesTy &&
9619 "If there are offloading arrays to pass to the target"
9620 "TaskTy cannot be the same as TaskWithPrivatesTy");
9621 (void)TaskTy;
9622 Value *Privates =
9623 Builder.CreateStructGEP(Ty: TaskWithPrivatesTy, Ptr: TaskWithPrivates, Idx: 1);
9624 for (unsigned int i = 0; i < NumOffloadingArrays; ++i)
9625 KernelLaunchArgs.push_back(
9626 Elt: Builder.CreateStructGEP(Ty: PrivatesTy, Ptr: Privates, Idx: i));
9627 }
9628
9629 if (HasShareds) {
9630 auto *ArgStructAlloca =
9631 dyn_cast<AllocaInst>(Val: StaleCI->getArgOperand(i: SharedArgsOperandNo));
9632 assert(ArgStructAlloca &&
9633 "Unable to find the alloca instruction corresponding to arguments "
9634 "for extracted function");
9635 auto *ArgStructType = cast<StructType>(Val: ArgStructAlloca->getAllocatedType());
9636 std::optional<TypeSize> ArgAllocSize =
9637 ArgStructAlloca->getAllocationSize(DL: M.getDataLayout());
9638 assert(ArgStructType && ArgAllocSize &&
9639 "Unable to determine size of arguments for extracted function");
9640 uint64_t StructSize = ArgAllocSize->getFixedValue();
9641
9642 AllocaInst *NewArgStructAlloca =
9643 Builder.CreateAlloca(Ty: ArgStructType, ArraySize: nullptr, Name: "structArg");
9644
9645 Value *SharedsSize = Builder.getInt64(C: StructSize);
9646
9647 LoadInst *LoadShared =
9648 loadSharedDataFromTaskDescriptor(Builder, TaskWithPrivates);
9649
9650 Builder.CreateMemCpy(
9651 Dst: NewArgStructAlloca, DstAlign: NewArgStructAlloca->getAlign(), Src: LoadShared,
9652 SrcAlign: LoadShared->getPointerAlignment(DL: M.getDataLayout()), Size: SharedsSize);
9653 KernelLaunchArgs.push_back(Elt: NewArgStructAlloca);
9654 }
9655 OMPBuilder.createRuntimeFunctionCall(Callee: KernelLaunchFunction, Args: KernelLaunchArgs);
9656 Builder.CreateRetVoid();
9657 return ProxyFn;
9658}
9659static Type *getOffloadingArrayType(Value *V) {
9660
9661 if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: V))
9662 return GEP->getSourceElementType();
9663 if (auto *Alloca = dyn_cast<AllocaInst>(Val: V))
9664 return Alloca->getAllocatedType();
9665
9666 llvm_unreachable("Unhandled Instruction type");
9667 return nullptr;
9668}
9669// This function returns a struct that has at most two members.
9670// The first member is always %struct.kmp_task_ompbuilder_t, that is the task
9671// descriptor. The second member, if needed, is a struct containing arrays
9672// that need to be passed to the offloaded target kernel. For example,
9673// if .offload_baseptrs, .offload_ptrs and .offload_sizes have to be passed to
9674// the target kernel and their types are [3 x ptr], [3 x ptr] and [3 x i64]
9675// respectively, then the types created by this function are
9676//
9677// %struct.privates = type { [3 x ptr], [3 x ptr], [3 x i64] }
9678// %struct.task_with_privates = type { %struct.kmp_task_ompbuilder_t,
9679// %struct.privates }
9680// %struct.task_with_privates is returned by this function.
9681// If there aren't any offloading arrays to pass to the target kernel,
9682// %struct.kmp_task_ompbuilder_t is returned.
9683static StructType *
9684createTaskWithPrivatesTy(OpenMPIRBuilder &OMPIRBuilder,
9685 ArrayRef<Value *> OffloadingArraysToPrivatize,
9686 ArrayRef<Type *> DescriptorTypes) {
9687
9688 if (OffloadingArraysToPrivatize.empty() && DescriptorTypes.empty())
9689 return OMPIRBuilder.Task;
9690
9691 SmallVector<Type *, 4> StructFieldTypes;
9692 for (Value *V : OffloadingArraysToPrivatize) {
9693 assert(V->getType()->isPointerTy() &&
9694 "Expected pointer to array to privatize. Got a non-pointer value "
9695 "instead");
9696 Type *ArrayTy = getOffloadingArrayType(V);
9697 assert(ArrayTy && "ArrayType cannot be nullptr");
9698 StructFieldTypes.push_back(Elt: ArrayTy);
9699 }
9700 llvm::append_range(C&: StructFieldTypes, R&: DescriptorTypes);
9701 StructType *PrivatesStructTy =
9702 StructType::create(Elements: StructFieldTypes, Name: "struct.privates");
9703 return StructType::create(Elements: {OMPIRBuilder.Task, PrivatesStructTy},
9704 Name: "struct.task_with_privates");
9705}
9706static Error emitTargetOutlinedFunction(
9707 OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, bool IsOffloadEntry,
9708 TargetRegionEntryInfo &EntryInfo,
9709 const OpenMPIRBuilder::TargetKernelDefaultAttrs &DefaultAttrs,
9710 Function *&OutlinedFn, Constant *&OutlinedFnID,
9711 SmallVectorImpl<Value *> &Inputs,
9712 OpenMPIRBuilder::TargetBodyGenCallbackTy &CBFunc,
9713 OpenMPIRBuilder::TargetGenArgAccessorsCallbackTy &ArgAccessorFuncCB,
9714 DebugLoc OutlinedFnLoc) {
9715
9716 OpenMPIRBuilder::FunctionGenCallback &&GenerateOutlinedFunction =
9717 [&](StringRef EntryFnName) {
9718 return createOutlinedFunction(OMPBuilder, Builder, DefaultAttrs,
9719 FuncName: EntryFnName, Inputs, CBFunc,
9720 ArgAccessorFuncCB, OutlinedFnLoc);
9721 };
9722
9723 return OMPBuilder.emitTargetRegionFunction(
9724 EntryInfo, GenerateFunctionCallback&: GenerateOutlinedFunction, IsOffloadEntry, OutlinedFn,
9725 OutlinedFnID);
9726}
9727
9728OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitTargetTask(
9729 TargetTaskBodyCallbackTy TaskBodyCB, Value *DeviceID, Value *RTLoc,
9730 OpenMPIRBuilder::InsertPointTy AllocaIP,
9731 const DependenciesInfo &Dependencies, const TargetDataRTArgs &RTArgs,
9732 bool HasNoWait) {
9733
9734 // The following explains the code-gen scenario for the `target` directive. A
9735 // similar scneario is followed for other device-related directives (e.g.
9736 // `target enter data`) but in similar fashion since we only need to emit task
9737 // that encapsulates the proper runtime call.
9738 //
9739 // When we arrive at this function, the target region itself has been
9740 // outlined into the function OutlinedFn.
9741 // So at ths point, for
9742 // --------------------------------------------------------------
9743 // void user_code_that_offloads(...) {
9744 // omp target depend(..) map(from:a) map(to:b) private(i)
9745 // do i = 1, 10
9746 // a(i) = b(i) + n
9747 // }
9748 //
9749 // --------------------------------------------------------------
9750 //
9751 // we have
9752 //
9753 // --------------------------------------------------------------
9754 //
9755 // void user_code_that_offloads(...) {
9756 // %.offload_baseptrs = alloca [2 x ptr], align 8
9757 // %.offload_ptrs = alloca [2 x ptr], align 8
9758 // %.offload_mappers = alloca [2 x ptr], align 8
9759 // ;; target region has been outlined and now we need to
9760 // ;; offload to it via a target task.
9761 // }
9762 // void outlined_device_function(ptr a, ptr b, ptr n) {
9763 // n = *n_ptr;
9764 // do i = 1, 10
9765 // a(i) = b(i) + n
9766 // }
9767 //
9768 // We have to now do the following
9769 // (i) Make an offloading call to outlined_device_function using the OpenMP
9770 // RTL. See 'kernel_launch_function' in the pseudo code below. This is
9771 // emitted by emitKernelLaunch
9772 // (ii) Create a task entry point function that calls kernel_launch_function
9773 // and is the entry point for the target task. See
9774 // '@.omp_target_task_proxy_func in the pseudocode below.
9775 // (iii) Create a task with the task entry point created in (ii)
9776 //
9777 // That is we create the following
9778 // struct task_with_privates {
9779 // struct kmp_task_ompbuilder_t task_struct;
9780 // struct privates {
9781 // [2 x ptr] ; baseptrs
9782 // [2 x ptr] ; ptrs
9783 // [2 x i64] ; sizes
9784 // }
9785 // }
9786 // void user_code_that_offloads(...) {
9787 // %.offload_baseptrs = alloca [2 x ptr], align 8
9788 // %.offload_ptrs = alloca [2 x ptr], align 8
9789 // %.offload_sizes = alloca [2 x i64], align 8
9790 //
9791 // %structArg = alloca { ptr, ptr, ptr }, align 8
9792 // %strucArg[0] = a
9793 // %strucArg[1] = b
9794 // %strucArg[2] = &n
9795 //
9796 // target_task_with_privates = @__kmpc_omp_target_task_alloc(...,
9797 // sizeof(kmp_task_ompbuilder_t),
9798 // sizeof(structArg),
9799 // @.omp_target_task_proxy_func,
9800 // ...)
9801 // memcpy(target_task_with_privates->task_struct->shareds, %structArg,
9802 // sizeof(structArg))
9803 // memcpy(target_task_with_privates->privates->baseptrs,
9804 // offload_baseptrs, sizeof(offload_baseptrs)
9805 // memcpy(target_task_with_privates->privates->ptrs,
9806 // offload_ptrs, sizeof(offload_ptrs)
9807 // memcpy(target_task_with_privates->privates->sizes,
9808 // offload_sizes, sizeof(offload_sizes)
9809 // dependencies_array = ...
9810 // ;; if nowait not present
9811 // call @__kmpc_omp_wait_deps(..., dependencies_array)
9812 // call @__kmpc_omp_task_begin_if0(...)
9813 // call @ @.omp_target_task_proxy_func(i32 thread_id, ptr
9814 // %target_task_with_privates)
9815 // call @__kmpc_omp_task_complete_if0(...)
9816 // }
9817 //
9818 // define internal void @.omp_target_task_proxy_func(i32 %thread.id,
9819 // ptr %task) {
9820 // %structArg = alloca {ptr, ptr, ptr}
9821 // %task_ptr = getelementptr(%task, 0, 0)
9822 // %shared_data = load (getelementptr %task_ptr, 0, 0)
9823 // mempcy(%structArg, %shared_data, sizeof(%structArg))
9824 //
9825 // %offloading_arrays = getelementptr(%task, 0, 1)
9826 // %offload_baseptrs = getelementptr(%offloading_arrays, 0, 0)
9827 // %offload_ptrs = getelementptr(%offloading_arrays, 0, 1)
9828 // %offload_sizes = getelementptr(%offloading_arrays, 0, 2)
9829 // kernel_launch_function(%thread.id, %offload_baseptrs, %offload_ptrs,
9830 // %offload_sizes, %structArg)
9831 // }
9832 //
9833 // We need the proxy function because the signature of the task entry point
9834 // expected by kmpc_omp_task is always the same and will be different from
9835 // that of the kernel_launch function.
9836 //
9837 // kernel_launch_function is generated by emitKernelLaunch and has the
9838 // always_inline attribute. For this example, it'll look like so:
9839 // void kernel_launch_function(%thread_id, %offload_baseptrs, %offload_ptrs,
9840 // %offload_sizes, %structArg) alwaysinline {
9841 // %kernel_args = alloca %struct.__tgt_kernel_arguments, align 8
9842 // ; load aggregated data from %structArg
9843 // ; setup kernel_args using offload_baseptrs, offload_ptrs and
9844 // ; offload_sizes
9845 // call i32 @__tgt_target_kernel(...,
9846 // outlined_device_function,
9847 // ptr %kernel_args)
9848 // }
9849 // void outlined_device_function(ptr a, ptr b, ptr n) {
9850 // n = *n_ptr;
9851 // do i = 1, 10
9852 // a(i) = b(i) + n
9853 // }
9854 //
9855 BasicBlock *TargetTaskBodyBB =
9856 splitBB(Builder, /*CreateBranch=*/true, Name: "target.task.body");
9857 BasicBlock *TargetTaskAllocaBB =
9858 splitBB(Builder, /*CreateBranch=*/true, Name: "target.task.alloca");
9859
9860 InsertPointTy TargetTaskAllocaIP(TargetTaskAllocaBB->begin());
9861 InsertPointTy TargetTaskBodyIP(TargetTaskBodyBB->begin());
9862
9863 auto OI = std::make_unique<OutlineInfo>();
9864 OI->EntryBB = TargetTaskAllocaBB;
9865 OI->OuterAllocBB = AllocaIP.getNodeParent();
9866
9867 // Add the thread ID argument.
9868 SmallVector<Instruction *, 4> ToBeDeleted;
9869 OI->ExcludeArgsFromAggregate.push_back(Elt: createFakeIntVal(
9870 Builder, OuterAllocaIP: AllocaIP, ToBeDeleted, InnerAllocaIP: TargetTaskAllocaIP, Name: "global.tid", AsPtr: false));
9871
9872 // Generate the task body which will subsequently be outlined.
9873 Builder.restoreIP(IP: TargetTaskBodyIP);
9874 if (Error Err = TaskBodyCB(DeviceID, RTLoc, TargetTaskAllocaIP))
9875 return Err;
9876
9877 // The outliner (CodeExtractor) extract a sequence or vector of blocks that
9878 // it is given. These blocks are enumerated by
9879 // OpenMPIRBuilder::OutlineInfo::collectBlocks which expects the OI.ExitBlock
9880 // to be outside the region. In other words, OI.ExitBlock is expected to be
9881 // the start of the region after the outlining. We used to set OI.ExitBlock
9882 // to the InsertBlock after TaskBodyCB is done. This is fine in most cases
9883 // except when the task body is a single basic block. In that case,
9884 // OI.ExitBlock is set to the single task body block and will get left out of
9885 // the outlining process. So, simply create a new empty block to which we
9886 // uncoditionally branch from where TaskBodyCB left off
9887 OI->ExitBB = BasicBlock::Create(Context&: Builder.getContext(), Name: "target.task.cont");
9888 emitBlock(BB: OI->ExitBB, CurFn: Builder.GetInsertBlock()->getParent(),
9889 /*IsFinished=*/true);
9890
9891 SmallVector<Value *, 2> OffloadingArraysToPrivatize;
9892 bool NeedsTargetTask = HasNoWait && DeviceID;
9893 if (NeedsTargetTask) {
9894 for (auto *V :
9895 {RTArgs.BasePointersArray, RTArgs.PointersArray, RTArgs.MappersArray,
9896 RTArgs.MapNamesArray, RTArgs.MapTypesArray, RTArgs.MapTypesArrayEnd,
9897 RTArgs.SizesArray}) {
9898 if (V && !isa<ConstantPointerNull, GlobalVariable>(Val: V)) {
9899 OffloadingArraysToPrivatize.push_back(Elt: V);
9900 OI->ExcludeArgsFromAggregate.push_back(Elt: V);
9901 }
9902 }
9903 }
9904 auto NonContiguousDescriptors = RTArgs.NonContiguousDescriptors;
9905 if (!NeedsTargetTask)
9906 NonContiguousDescriptors.clear();
9907 OI->PostOutlineCB = [this, ToBeDeleted, Dependencies, NeedsTargetTask,
9908 DeviceID, OffloadingArraysToPrivatize,
9909 NonContiguousDescriptors,
9910 PointersArray =
9911 RTArgs.PointersArray](Function &OutlinedFn) mutable {
9912 assert(OutlinedFn.hasOneUse() &&
9913 "there must be a single user for the outlined function");
9914
9915 CallInst *StaleCI = cast<CallInst>(Val: OutlinedFn.user_back());
9916
9917 // The first argument of StaleCI is always the thread id.
9918 // The next few arguments are the pointers to offloading arrays
9919 // if any. (see OffloadingArraysToPrivatize)
9920 // Finally, all other local values that are live-in into the outlined region
9921 // end up in a structure whose pointer is passed as the last argument. This
9922 // piece of data is passed in the "shared" field of the task structure. So,
9923 // we know we have to pass shareds to the task if the number of arguments is
9924 // greater than OffloadingArraysToPrivatize.size() + 1 The 1 is for the
9925 // thread id. Further, for safety, we assert that the number of arguments of
9926 // StaleCI is exactly OffloadingArraysToPrivatize.size() + 2
9927 const unsigned int NumStaleCIArgs = StaleCI->arg_size();
9928 bool HasShareds = NumStaleCIArgs > OffloadingArraysToPrivatize.size() + 1;
9929 assert((!HasShareds ||
9930 NumStaleCIArgs == (OffloadingArraysToPrivatize.size() + 2)) &&
9931 "Wrong number of arguments for StaleCI when shareds are present");
9932 int SharedArgOperandNo =
9933 HasShareds ? OffloadingArraysToPrivatize.size() + 1 : 0;
9934
9935 // Descriptor storage is not an argument to the outlined function. It is
9936 // reached indirectly through the task-private pointer array.
9937 SmallVector<Type *, 2> DescriptorTypes;
9938 for (const auto &Descriptor : NonContiguousDescriptors)
9939 DescriptorTypes.push_back(Elt: Descriptor.Storage->getAllocatedType());
9940 StructType *TaskWithPrivatesTy = createTaskWithPrivatesTy(
9941 OMPIRBuilder&: *this, OffloadingArraysToPrivatize, DescriptorTypes);
9942 StructType *PrivatesTy = nullptr;
9943
9944 if (!OffloadingArraysToPrivatize.empty())
9945 PrivatesTy =
9946 static_cast<StructType *>(TaskWithPrivatesTy->getElementType(N: 1));
9947
9948 Function *ProxyFn = emitTargetTaskProxyFunction(
9949 OMPBuilder&: *this, Builder, StaleCI, PrivatesTy, TaskWithPrivatesTy,
9950 NumOffloadingArrays: OffloadingArraysToPrivatize.size(), SharedArgsOperandNo: SharedArgOperandNo);
9951
9952 LLVM_DEBUG(dbgs() << "Proxy task entry function created: " << *ProxyFn
9953 << "\n");
9954
9955 Builder.SetInsertPoint(StaleCI);
9956
9957 // Gather the arguments for emitting the runtime call.
9958 uint32_t SrcLocStrSize;
9959 Constant *SrcLocStr =
9960 getOrCreateSrcLocStr(Loc: LocationDescription(Builder), SrcLocStrSize);
9961 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
9962
9963 // @__kmpc_omp_task_alloc or @__kmpc_omp_target_task_alloc
9964 //
9965 // If `HasNoWait == true`, we call @__kmpc_omp_target_task_alloc to provide
9966 // the DeviceID to the deferred task and also since
9967 // @__kmpc_omp_target_task_alloc creates an untied/async task.
9968 Function *TaskAllocFn =
9969 !NeedsTargetTask
9970 ? getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_alloc)
9971 : getOrCreateRuntimeFunctionPtr(
9972 FnID: OMPRTL___kmpc_omp_target_task_alloc);
9973
9974 // Arguments - `loc_ref` (Ident) and `gtid` (ThreadID)
9975 // call.
9976 Value *ThreadID = getOrCreateThreadID(Ident);
9977
9978 // Argument - `sizeof_kmp_task_t` (TaskSize)
9979 // Tasksize refers to the size in bytes of kmp_task_t data structure
9980 // plus any other data to be passed to the target task, if any, which
9981 // is packed into a struct. kmp_task_t and the struct so created are
9982 // packed into a wrapper struct whose type is TaskWithPrivatesTy.
9983 Value *TaskSize = Builder.getInt64(
9984 C: M.getDataLayout().getTypeStoreSize(Ty: TaskWithPrivatesTy));
9985
9986 // Argument - `sizeof_shareds` (SharedsSize)
9987 // SharedsSize refers to the shareds array size in the kmp_task_t data
9988 // structure.
9989 Value *SharedsSize = Builder.getInt64(C: 0);
9990 if (HasShareds) {
9991 auto *ArgStructAlloca =
9992 dyn_cast<AllocaInst>(Val: StaleCI->getArgOperand(i: SharedArgOperandNo));
9993 assert(ArgStructAlloca &&
9994 "Unable to find the alloca instruction corresponding to arguments "
9995 "for extracted function");
9996 std::optional<TypeSize> ArgAllocSize =
9997 ArgStructAlloca->getAllocationSize(DL: M.getDataLayout());
9998 assert(ArgAllocSize &&
9999 "Unable to determine size of arguments for extracted function");
10000 SharedsSize = Builder.getInt64(C: ArgAllocSize->getFixedValue());
10001 }
10002
10003 // Argument - `flags`
10004 // Task is tied iff (Flags & 1) == 1.
10005 // Task is untied iff (Flags & 1) == 0.
10006 // Task is final iff (Flags & 2) == 2.
10007 // Task is not final iff (Flags & 2) == 0.
10008 // A target task is not final and is untied.
10009 Value *Flags = Builder.getInt32(C: 0);
10010
10011 // Emit the @__kmpc_omp_task_alloc runtime call
10012 // The runtime call returns a pointer to an area where the task captured
10013 // variables must be copied before the task is run (TaskData)
10014 CallInst *TaskData = nullptr;
10015
10016 SmallVector<llvm::Value *> TaskAllocArgs = {
10017 /*loc_ref=*/Ident, /*gtid=*/ThreadID,
10018 /*flags=*/Flags,
10019 /*sizeof_task=*/TaskSize, /*sizeof_shared=*/SharedsSize,
10020 /*task_func=*/ProxyFn};
10021
10022 if (NeedsTargetTask) {
10023 assert(DeviceID && "Expected non-empty device ID.");
10024 TaskAllocArgs.push_back(Elt: DeviceID);
10025 }
10026
10027 TaskData = createRuntimeFunctionCall(Callee: TaskAllocFn, Args: TaskAllocArgs);
10028
10029 Align Alignment = TaskData->getPointerAlignment(DL: M.getDataLayout());
10030 if (HasShareds) {
10031 Value *Shareds = StaleCI->getArgOperand(i: SharedArgOperandNo);
10032 Value *TaskShareds = loadSharedDataFromTaskDescriptor(Builder, TaskWithPrivates: TaskData);
10033 Builder.CreateMemCpy(Dst: TaskShareds, DstAlign: Alignment, Src: Shareds, SrcAlign: Alignment,
10034 Size: SharedsSize);
10035 }
10036 if (!OffloadingArraysToPrivatize.empty()) {
10037 Value *Privates =
10038 Builder.CreateStructGEP(Ty: TaskWithPrivatesTy, Ptr: TaskData, Idx: 1);
10039 Value *PrivatePointersArray = nullptr;
10040 for (unsigned int i = 0; i < OffloadingArraysToPrivatize.size(); ++i) {
10041 Value *PtrToPrivatize = OffloadingArraysToPrivatize[i];
10042 [[maybe_unused]] Type *ArrayType =
10043 getOffloadingArrayType(V: PtrToPrivatize);
10044 assert(ArrayType && "ArrayType cannot be nullptr");
10045
10046 Type *ElementType = PrivatesTy->getElementType(N: i);
10047 assert(ElementType == ArrayType &&
10048 "ElementType should match ArrayType");
10049 (void)ArrayType;
10050
10051 Value *Dst = Builder.CreateStructGEP(Ty: PrivatesTy, Ptr: Privates, Idx: i);
10052 Builder.CreateMemCpy(
10053 Dst, DstAlign: Alignment, Src: PtrToPrivatize, SrcAlign: Alignment,
10054 Size: Builder.getInt64(C: M.getDataLayout().getTypeStoreSize(Ty: ElementType)));
10055 if (PtrToPrivatize == PointersArray)
10056 PrivatePointersArray = Dst;
10057 }
10058 // Snapshot the encounter-time bounds before enqueueing the task, then
10059 // redirect its pointer entries to descriptors owned by that task.
10060 for (auto [I, Descriptor] : llvm::enumerate(First&: NonContiguousDescriptors)) {
10061 assert(PrivatePointersArray && "missing task-private pointer array");
10062 Value *Dst = Builder.CreateStructGEP(
10063 Ty: PrivatesTy, Ptr: Privates, Idx: OffloadingArraysToPrivatize.size() + I);
10064 Builder.CreateMemCpy(
10065 Dst, DstAlign: Alignment, Src: Descriptor.Storage, SrcAlign: Descriptor.Storage->getAlign(),
10066 Size: Builder.getInt64(C: M.getDataLayout().getTypeStoreSize(
10067 Ty: Descriptor.Storage->getAllocatedType())));
10068 Value *Entry = Builder.CreateConstInBoundsGEP2_32(
10069 Ty: getOffloadingArrayType(V: PointersArray), Ptr: PrivatePointersArray, Idx0: 0,
10070 Idx1: Descriptor.MapIndex);
10071 Builder.CreateAlignedStore(Val: Dst, Ptr: Entry, Align: Alignment);
10072 }
10073 }
10074
10075 Value *DepArray = nullptr;
10076 Value *NumDeps = nullptr;
10077 if (Dependencies.DepArray) {
10078 DepArray = Dependencies.DepArray;
10079 NumDeps = Dependencies.NumDeps;
10080 } else if (!Dependencies.Deps.empty()) {
10081 DepArray = emitTaskDependencies(OMPBuilder&: *this, Dependencies: Dependencies.Deps);
10082 NumDeps = Builder.getInt32(C: Dependencies.Deps.size());
10083 }
10084
10085 // ---------------------------------------------------------------
10086 // V5.2 13.8 target construct
10087 // If the nowait clause is present, execution of the target task
10088 // may be deferred. If the nowait clause is not present, the target task is
10089 // an included task.
10090 // ---------------------------------------------------------------
10091 // The above means that the lack of a nowait on the target construct
10092 // translates to '#pragma omp task if(0)'
10093 if (!NeedsTargetTask) {
10094 if (DepArray) {
10095 Function *TaskWaitFn =
10096 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_wait_deps);
10097 createRuntimeFunctionCall(
10098 Callee: TaskWaitFn,
10099 Args: {/*loc_ref=*/Ident, /*gtid=*/ThreadID,
10100 /*ndeps=*/NumDeps,
10101 /*dep_list=*/DepArray,
10102 /*ndeps_noalias=*/ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0),
10103 /*noalias_dep_list=*/
10104 ConstantPointerNull::get(T: PointerType::getUnqual(C&: M.getContext()))});
10105 }
10106 // Included task.
10107 Function *TaskBeginFn =
10108 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_begin_if0);
10109 Function *TaskCompleteFn =
10110 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_complete_if0);
10111 createRuntimeFunctionCall(Callee: TaskBeginFn, Args: {Ident, ThreadID, TaskData});
10112 CallInst *CI = createRuntimeFunctionCall(Callee: ProxyFn, Args: {ThreadID, TaskData});
10113 CI->setDebugLoc(StaleCI->getDebugLoc());
10114 createRuntimeFunctionCall(Callee: TaskCompleteFn, Args: {Ident, ThreadID, TaskData});
10115 } else if (DepArray) {
10116 // HasNoWait - meaning the task may be deferred. Call
10117 // __kmpc_omp_task_with_deps if there are dependencies,
10118 // else call __kmpc_omp_task
10119 Function *TaskFn =
10120 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_with_deps);
10121 createRuntimeFunctionCall(
10122 Callee: TaskFn,
10123 Args: {Ident, ThreadID, TaskData, NumDeps, DepArray,
10124 ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0),
10125 ConstantPointerNull::get(T: PointerType::getUnqual(C&: M.getContext()))});
10126 } else {
10127 // Emit the @__kmpc_omp_task runtime call to spawn the task
10128 Function *TaskFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task);
10129 createRuntimeFunctionCall(Callee: TaskFn, Args: {Ident, ThreadID, TaskData});
10130 }
10131
10132 Builder.ClearInsertionPoint();
10133 StaleCI->eraseFromParent();
10134 for (Instruction *I : llvm::reverse(C&: ToBeDeleted))
10135 I->eraseFromParent();
10136 };
10137 addOutlineInfo(OI: std::move(OI));
10138
10139 LLVM_DEBUG(dbgs() << "Insert block after emitKernelLaunch = \n"
10140 << *(Builder.GetInsertBlock()) << "\n");
10141 LLVM_DEBUG(dbgs() << "Module after emitKernelLaunch = \n"
10142 << *(Builder.getModule()) << "\n");
10143 return Builder.saveIP();
10144}
10145
10146Error OpenMPIRBuilder::emitOffloadingArraysAndArgs(
10147 InsertPointTy AllocaIP, InsertPointTy CodeGenIP, TargetDataInfo &Info,
10148 TargetDataRTArgs &RTArgs, MapInfosTy &CombinedInfo,
10149 CustomMapperCallbackTy CustomMapperCB, bool IsNonContiguous,
10150 bool ForEndCall, function_ref<void(unsigned int, Value *)> DeviceAddrCB) {
10151 if (Error Err =
10152 emitOffloadingArrays(AllocaIP, CodeGenIP, CombinedInfo, Info,
10153 CustomMapperCB, IsNonContiguous, DeviceAddrCB))
10154 return Err;
10155 emitOffloadingArraysArgument(Builder, RTArgs, Info, ForEndCall);
10156 return Error::success();
10157}
10158
10159static Error emitTargetCall(
10160 OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, Value *RTLocOverride,
10161 OpenMPIRBuilder::InsertPointTy AllocaIP,
10162 ArrayRef<BasicBlock *> DeallocBlocks, OpenMPIRBuilder::TargetDataInfo &Info,
10163 const OpenMPIRBuilder::TargetKernelDefaultAttrs &DefaultAttrs,
10164 const OpenMPIRBuilder::TargetKernelRuntimeAttrs &RuntimeAttrs,
10165 Value *IfCond, Function *OutlinedFn, Constant *OutlinedFnID,
10166 SmallVectorImpl<Value *> &Args,
10167 OpenMPIRBuilder::GenMapInfoCallbackTy GenMapInfoCB,
10168 OpenMPIRBuilder::CustomMapperCallbackTy CustomMapperCB,
10169 const OpenMPIRBuilder::DependenciesInfo &Dependencies, bool HasNoWait,
10170 Value *DynCGroupMem, OMPDynGroupprivateFallbackType DynCGroupMemFallback) {
10171 // Generate a function call to the host fallback implementation of the target
10172 // region. This is called by the host when no offload entry was generated for
10173 // the target region and when the offloading call fails at runtime.
10174 auto &&EmitTargetCallFallbackCB = [&](OpenMPIRBuilder::InsertPointTy IP)
10175 -> OpenMPIRBuilder::InsertPointOrErrorTy {
10176 Builder.restoreIP(IP);
10177 // Ensure the host fallback has the same dyn_ptr ABI as the device.
10178 SmallVector<Value *> FallbackArgs(Args.begin(), Args.end());
10179 FallbackArgs.push_back(
10180 Elt: Constant::getNullValue(Ty: PointerType::getUnqual(C&: Builder.getContext())));
10181 OMPBuilder.createRuntimeFunctionCall(Callee: OutlinedFn, Args: FallbackArgs);
10182 return Builder.saveIP();
10183 };
10184
10185 bool HasDependencies = !Dependencies.empty();
10186 bool RequiresOuterTargetTask = HasNoWait || HasDependencies;
10187
10188 OpenMPIRBuilder::TargetKernelArgs KArgs;
10189
10190 auto TaskBodyCB =
10191 [&](Value *DeviceID, Value *RTLoc,
10192 IRBuilderBase::InsertPoint TargetTaskAllocaIP) -> Error {
10193 // Assume no error was returned because EmitTargetCallFallbackCB doesn't
10194 // produce any.
10195 llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(ValOrErr: [&]() {
10196 // emitKernelLaunch makes the necessary runtime call to offload the
10197 // kernel. We then outline all that code into a separate function
10198 // ('kernel_launch_function' in the pseudo code above). This function is
10199 // then called by the target task proxy function (see
10200 // '@.omp_target_task_proxy_func' in the pseudo code above)
10201 // "@.omp_target_task_proxy_func' is generated by
10202 // emitTargetTaskProxyFunction.
10203 if (OutlinedFnID && DeviceID)
10204 return OMPBuilder.emitKernelLaunch(Loc: Builder, OutlinedFnID,
10205 EmitTargetCallFallbackCB, Args&: KArgs,
10206 DeviceID, RTLoc, AllocaIP: TargetTaskAllocaIP);
10207
10208 // We only need to do the outlining if `DeviceID` is set to avoid calling
10209 // `emitKernelLaunch` if we want to code-gen for the host; e.g. if we are
10210 // generating the `else` branch of an `if` clause.
10211 //
10212 // When OutlinedFnID is set to nullptr, then it's not an offloading call.
10213 // In this case, we execute the host implementation directly.
10214 return EmitTargetCallFallbackCB(OMPBuilder.Builder.saveIP());
10215 }());
10216
10217 OMPBuilder.Builder.restoreIP(IP: AfterIP);
10218 return Error::success();
10219 };
10220
10221 auto &&EmitTargetCallElse =
10222 [&](OpenMPIRBuilder::InsertPointTy AllocaIP,
10223 OpenMPIRBuilder::InsertPointTy CodeGenIP,
10224 ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
10225 // Assume no error was returned because EmitTargetCallFallbackCB doesn't
10226 // produce any.
10227 OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(ValOrErr: [&]() {
10228 if (RequiresOuterTargetTask) {
10229 // Arguments that are intended to be directly forwarded to an
10230 // emitKernelLaunch call are pased as nullptr, since
10231 // OutlinedFnID=nullptr results in that call not being done.
10232 OpenMPIRBuilder::TargetDataRTArgs EmptyRTArgs;
10233 return OMPBuilder.emitTargetTask(TaskBodyCB, /*DeviceID=*/nullptr,
10234 /*RTLoc=*/nullptr, AllocaIP,
10235 Dependencies, RTArgs: EmptyRTArgs, HasNoWait);
10236 }
10237 return EmitTargetCallFallbackCB(Builder.saveIP());
10238 }());
10239
10240 Builder.restoreIP(IP: AfterIP);
10241 return Error::success();
10242 };
10243
10244 auto &&EmitTargetCallThen =
10245 [&](OpenMPIRBuilder::InsertPointTy AllocaIP,
10246 OpenMPIRBuilder::InsertPointTy CodeGenIP,
10247 ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
10248 Info.HasNoWait = HasNoWait;
10249 OpenMPIRBuilder::MapInfosTy &MapInfo = GenMapInfoCB(Builder.saveIP());
10250
10251 OpenMPIRBuilder::TargetDataRTArgs RTArgs;
10252 if (Error Err = OMPBuilder.emitOffloadingArraysAndArgs(
10253 AllocaIP, CodeGenIP: Builder.saveIP(), Info, RTArgs, CombinedInfo&: MapInfo, CustomMapperCB,
10254 /*IsNonContiguous=*/true,
10255 /*ForEndCall=*/false))
10256 return Err;
10257
10258 SmallVector<Value *, 3> NumTeamsC;
10259 for (auto [DefaultVal, RuntimeVal] :
10260 zip_equal(t: DefaultAttrs.MaxTeams, u: RuntimeAttrs.MaxTeams))
10261 NumTeamsC.push_back(Elt: RuntimeVal ? RuntimeVal
10262 : Builder.getInt32(C: DefaultVal));
10263
10264 // Calculate number of threads: 0 if no clauses specified, otherwise it is
10265 // the minimum between optional THREAD_LIMIT and NUM_THREADS clauses.
10266 auto InitMaxThreadsClause = [&Builder](Value *Clause) {
10267 if (Clause)
10268 Clause = Builder.CreateIntCast(V: Clause, DestTy: Builder.getInt32Ty(),
10269 /*isSigned=*/false);
10270 return Clause;
10271 };
10272 auto CombineMaxThreadsClauses = [&Builder](Value *Clause, Value *&Result) {
10273 if (Clause)
10274 Result =
10275 Result ? Builder.CreateSelect(C: Builder.CreateICmpULT(LHS: Result, RHS: Clause),
10276 True: Result, False: Clause)
10277 : Clause;
10278 };
10279
10280 // If a multi-dimensional THREAD_LIMIT is set, it is the OMPX_BARE case, so
10281 // the NUM_THREADS clause is overriden by THREAD_LIMIT.
10282 SmallVector<Value *, 3> NumThreadsC;
10283 Value *MaxThreadsClause =
10284 RuntimeAttrs.TeamsThreadLimit.size() == 1
10285 ? InitMaxThreadsClause(RuntimeAttrs.MaxThreads.front())
10286 : nullptr;
10287
10288 for (auto [TeamsVal, TargetVal] : zip_equal(
10289 t: RuntimeAttrs.TeamsThreadLimit, u: RuntimeAttrs.TargetThreadLimit)) {
10290 Value *TeamsThreadLimitClause = InitMaxThreadsClause(TeamsVal);
10291 Value *NumThreads = InitMaxThreadsClause(TargetVal);
10292
10293 CombineMaxThreadsClauses(TeamsThreadLimitClause, NumThreads);
10294 CombineMaxThreadsClauses(MaxThreadsClause, NumThreads);
10295
10296 NumThreadsC.push_back(Elt: NumThreads ? NumThreads : Builder.getInt32(C: 0));
10297 }
10298
10299 unsigned NumTargetItems = Info.NumberOfPtrs;
10300 Value *RTLoc = RTLocOverride;
10301 if (!RTLoc) {
10302 uint32_t SrcLocStrSize;
10303 Constant *SrcLocStr =
10304 OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
10305 RTLoc = OMPBuilder.getOrCreateIdent(SrcLocStr, SrcLocStrSize,
10306 LocFlags: llvm::omp::IdentFlag(0), Reserve2Flags: 0);
10307 }
10308
10309 Value *TripCount = RuntimeAttrs.LoopTripCount
10310 ? Builder.CreateIntCast(V: RuntimeAttrs.LoopTripCount,
10311 DestTy: Builder.getInt64Ty(),
10312 /*isSigned=*/false)
10313 : Builder.getInt64(C: 0);
10314
10315 // Request zero groupprivate bytes by default.
10316 if (!DynCGroupMem)
10317 DynCGroupMem = Builder.getInt32(C: 0);
10318
10319 KArgs = OpenMPIRBuilder::TargetKernelArgs(
10320 NumTargetItems, RTArgs, TripCount, NumTeamsC, NumThreadsC, DynCGroupMem,
10321 HasNoWait, /*StrictBlocks=*/false, /*StrictThreads=*/false,
10322 DynCGroupMemFallback);
10323
10324 // Assume no error was returned because TaskBodyCB and
10325 // EmitTargetCallFallbackCB don't produce any.
10326 OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(ValOrErr: [&]() {
10327 // The presence of certain clauses on the target directive require the
10328 // explicit generation of the target task.
10329 if (RequiresOuterTargetTask)
10330 return OMPBuilder.emitTargetTask(TaskBodyCB, DeviceID: RuntimeAttrs.DeviceID,
10331 RTLoc, AllocaIP, Dependencies,
10332 RTArgs: KArgs.RTArgs, HasNoWait: Info.HasNoWait);
10333
10334 return OMPBuilder.emitKernelLaunch(
10335 Loc: Builder, OutlinedFnID, EmitTargetCallFallbackCB, Args&: KArgs,
10336 DeviceID: RuntimeAttrs.DeviceID, RTLoc, AllocaIP);
10337 }());
10338
10339 Builder.restoreIP(IP: AfterIP);
10340 return Error::success();
10341 };
10342
10343 // If we don't have an ID for the target region, it means an offload entry
10344 // wasn't created. In this case we just run the host fallback directly and
10345 // ignore any potential 'if' clauses.
10346 if (!OutlinedFnID)
10347 return EmitTargetCallElse(AllocaIP, Builder.saveIP(), DeallocBlocks);
10348
10349 // If there's no 'if' clause, only generate the kernel launch code path.
10350 if (!IfCond)
10351 return EmitTargetCallThen(AllocaIP, Builder.saveIP(), DeallocBlocks);
10352
10353 return OMPBuilder.emitIfClause(Cond: IfCond, ThenGen: EmitTargetCallThen, ElseGen: EmitTargetCallElse,
10354 AllocaIP);
10355}
10356
10357OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTarget(
10358 const LocationDescription &Loc, bool IsOffloadEntry, InsertPointTy AllocaIP,
10359 InsertPointTy CodeGenIP, ArrayRef<BasicBlock *> DeallocBlocks,
10360 TargetDataInfo &Info, TargetRegionEntryInfo &EntryInfo,
10361 const TargetKernelDefaultAttrs &DefaultAttrs,
10362 const TargetKernelRuntimeAttrs &RuntimeAttrs, Value *IfCond,
10363 SmallVectorImpl<Value *> &Inputs, GenMapInfoCallbackTy GenMapInfoCB,
10364 OpenMPIRBuilder::TargetBodyGenCallbackTy CBFunc,
10365 OpenMPIRBuilder::TargetGenArgAccessorsCallbackTy ArgAccessorFuncCB,
10366 CustomMapperCallbackTy CustomMapperCB, const DependenciesInfo &Dependencies,
10367 bool HasNowait, Value *DynCGroupMem,
10368 OMPDynGroupprivateFallbackType DynCGroupMemFallback, DebugLoc OutlinedFnLoc,
10369 Value *RTLocOverride) {
10370
10371 if (!updateToLocation(Loc))
10372 return InsertPointTy();
10373
10374 Builder.restoreIP(IP: CodeGenIP);
10375
10376 Function *OutlinedFn;
10377 Constant *OutlinedFnID = nullptr;
10378 // The target region is outlined into its own function. The LLVM IR for
10379 // the target region itself is generated using the callbacks CBFunc
10380 // and ArgAccessorFuncCB
10381 if (Error Err = emitTargetOutlinedFunction(
10382 OMPBuilder&: *this, Builder, IsOffloadEntry, EntryInfo, DefaultAttrs, OutlinedFn,
10383 OutlinedFnID, Inputs, CBFunc, ArgAccessorFuncCB, OutlinedFnLoc))
10384 return Err;
10385
10386 // If we are not on the target device, then we need to generate code
10387 // to make a remote call (offload) to the previously outlined function
10388 // that represents the target region. Do that now.
10389 if (!Config.isTargetDevice()) {
10390 if (Error Err = emitTargetCall(
10391 OMPBuilder&: *this, Builder, RTLocOverride, AllocaIP, DeallocBlocks, Info,
10392 DefaultAttrs, RuntimeAttrs, IfCond, OutlinedFn, OutlinedFnID,
10393 Args&: Inputs, GenMapInfoCB, CustomMapperCB, Dependencies, HasNoWait: HasNowait,
10394 DynCGroupMem, DynCGroupMemFallback))
10395 return Err;
10396 }
10397 return Builder.saveIP();
10398}
10399
10400std::string OpenMPIRBuilder::getNameWithSeparators(ArrayRef<StringRef> Parts,
10401 StringRef FirstSeparator,
10402 StringRef Separator) {
10403 SmallString<128> Buffer;
10404 llvm::raw_svector_ostream OS(Buffer);
10405 StringRef Sep = FirstSeparator;
10406 for (StringRef Part : Parts) {
10407 OS << Sep << Part;
10408 Sep = Separator;
10409 }
10410 return OS.str().str();
10411}
10412
10413std::string
10414OpenMPIRBuilder::createPlatformSpecificName(ArrayRef<StringRef> Parts) const {
10415 return OpenMPIRBuilder::getNameWithSeparators(Parts, FirstSeparator: Config.firstSeparator(),
10416 Separator: Config.separator());
10417}
10418
10419GlobalVariable *OpenMPIRBuilder::getOrCreateInternalVariable(
10420 Type *Ty, const StringRef &Name, std::optional<unsigned> AddressSpace) {
10421 auto &Elem = *InternalVars.try_emplace(Key: Name, Args: nullptr).first;
10422 if (Elem.second) {
10423 assert(Elem.second->getValueType() == Ty &&
10424 "OMP internal variable has different type than requested");
10425 } else {
10426 // TODO: investigate the appropriate linkage type used for the global
10427 // variable for possibly changing that to internal or private, or maybe
10428 // create different versions of the function for different OMP internal
10429 // variables.
10430 const DataLayout &DL = M.getDataLayout();
10431 // TODO: Investigate why AMDGPU expects AS 0 for globals even though the
10432 // default global AS is 1.
10433 // See double-target-call-with-declare-target.f90 and
10434 // declare-target-vars-in-target-region.f90 libomptarget
10435 // tests.
10436 unsigned AddressSpaceVal = AddressSpace ? *AddressSpace
10437 : M.getTargetTriple().isAMDGPU()
10438 ? 0
10439 : DL.getDefaultGlobalsAddressSpace();
10440 auto Linkage = this->M.getTargetTriple().isWasm()
10441 ? GlobalValue::InternalLinkage
10442 : GlobalValue::CommonLinkage;
10443 auto *GV = new GlobalVariable(M, Ty, /*IsConstant=*/false, Linkage,
10444 Constant::getNullValue(Ty), Elem.first(),
10445 /*InsertBefore=*/nullptr,
10446 GlobalValue::NotThreadLocal, AddressSpaceVal);
10447 const llvm::Align TypeAlign = DL.getABITypeAlign(Ty);
10448 const llvm::Align PtrAlign = DL.getPointerABIAlignment(AS: AddressSpaceVal);
10449 GV->setAlignment(std::max(a: TypeAlign, b: PtrAlign));
10450 Elem.second = GV;
10451 }
10452
10453 return Elem.second;
10454}
10455
10456Value *OpenMPIRBuilder::getOMPCriticalRegionLock(StringRef CriticalName) {
10457 std::string Prefix = Twine("gomp_critical_user_", CriticalName).str();
10458 std::string Name = getNameWithSeparators(Parts: {Prefix, "var"}, FirstSeparator: ".", Separator: ".");
10459 return getOrCreateInternalVariable(Ty: KmpCriticalNameTy, Name);
10460}
10461
10462Value *OpenMPIRBuilder::getSizeInBytes(Value *BasePtr) {
10463 LLVMContext &Ctx = Builder.getContext();
10464 Value *Null =
10465 Constant::getNullValue(Ty: PointerType::getUnqual(C&: BasePtr->getContext()));
10466 Value *SizeGep =
10467 Builder.CreateGEP(Ty: BasePtr->getType(), Ptr: Null, IdxList: Builder.getInt32(C: 1));
10468 Value *SizePtrToInt = Builder.CreatePtrToInt(V: SizeGep, DestTy: Type::getInt64Ty(C&: Ctx));
10469 return SizePtrToInt;
10470}
10471
10472GlobalVariable *
10473OpenMPIRBuilder::createOffloadMaptypes(SmallVectorImpl<uint64_t> &Mappings,
10474 std::string VarName) {
10475 llvm::Constant *MaptypesArrayInit =
10476 llvm::ConstantDataArray::get(Context&: M.getContext(), Elts&: Mappings);
10477 auto *MaptypesArrayGlobal = new llvm::GlobalVariable(
10478 M, MaptypesArrayInit->getType(),
10479 /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage, MaptypesArrayInit,
10480 VarName);
10481 MaptypesArrayGlobal->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
10482 return MaptypesArrayGlobal;
10483}
10484
10485void OpenMPIRBuilder::createMapperAllocas(const LocationDescription &Loc,
10486 InsertPointTy AllocaIP,
10487 unsigned NumOperands,
10488 struct MapperAllocas &MapperAllocas) {
10489 if (!updateToLocation(Loc))
10490 return;
10491
10492 auto *ArrI8PtrTy = ArrayType::get(ElementType: Int8Ptr, NumElements: NumOperands);
10493 auto *ArrI64Ty = ArrayType::get(ElementType: Int64, NumElements: NumOperands);
10494 Builder.restoreIP(IP: AllocaIP);
10495 AllocaInst *ArgsBase = Builder.CreateAlloca(
10496 Ty: ArrI8PtrTy, /* ArraySize = */ nullptr, Name: ".offload_baseptrs");
10497 AllocaInst *Args = Builder.CreateAlloca(Ty: ArrI8PtrTy, /* ArraySize = */ nullptr,
10498 Name: ".offload_ptrs");
10499 AllocaInst *ArgSizes = Builder.CreateAlloca(
10500 Ty: ArrI64Ty, /* ArraySize = */ nullptr, Name: ".offload_sizes");
10501 updateToLocation(Loc);
10502 MapperAllocas.ArgsBase = ArgsBase;
10503 MapperAllocas.Args = Args;
10504 MapperAllocas.ArgSizes = ArgSizes;
10505}
10506
10507void OpenMPIRBuilder::emitMapperCall(const LocationDescription &Loc,
10508 Function *MapperFunc, Value *SrcLocInfo,
10509 Value *MaptypesArg, Value *MapnamesArg,
10510 struct MapperAllocas &MapperAllocas,
10511 int64_t DeviceID, unsigned NumOperands) {
10512 if (!updateToLocation(Loc))
10513 return;
10514
10515 auto *ArrI8PtrTy = ArrayType::get(ElementType: Int8Ptr, NumElements: NumOperands);
10516 auto *ArrI64Ty = ArrayType::get(ElementType: Int64, NumElements: NumOperands);
10517 Value *ArgsBaseGEP =
10518 Builder.CreateInBoundsGEP(Ty: ArrI8PtrTy, Ptr: MapperAllocas.ArgsBase,
10519 IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: 0)});
10520 Value *ArgsGEP =
10521 Builder.CreateInBoundsGEP(Ty: ArrI8PtrTy, Ptr: MapperAllocas.Args,
10522 IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: 0)});
10523 Value *ArgSizesGEP =
10524 Builder.CreateInBoundsGEP(Ty: ArrI64Ty, Ptr: MapperAllocas.ArgSizes,
10525 IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: 0)});
10526 Value *NullPtr =
10527 Constant::getNullValue(Ty: PointerType::getUnqual(C&: Int8Ptr->getContext()));
10528 createRuntimeFunctionCall(Callee: MapperFunc, Args: {SrcLocInfo, Builder.getInt64(C: DeviceID),
10529 Builder.getInt32(C: NumOperands),
10530 ArgsBaseGEP, ArgsGEP, ArgSizesGEP,
10531 MaptypesArg, MapnamesArg, NullPtr});
10532}
10533
10534void OpenMPIRBuilder::emitOffloadingArraysArgument(IRBuilderBase &Builder,
10535 TargetDataRTArgs &RTArgs,
10536 TargetDataInfo &Info,
10537 bool ForEndCall) {
10538 assert((!ForEndCall || Info.separateBeginEndCalls()) &&
10539 "expected region end call to runtime only when end call is separate");
10540 auto UnqualPtrTy = PointerType::getUnqual(C&: M.getContext());
10541 auto VoidPtrTy = UnqualPtrTy;
10542 auto VoidPtrPtrTy = UnqualPtrTy;
10543 auto Int64Ty = Type::getInt64Ty(C&: M.getContext());
10544 auto Int64PtrTy = UnqualPtrTy;
10545
10546 RTArgs.NonContiguousDescriptors = Info.RTArgs.NonContiguousDescriptors;
10547 if (!Info.NumberOfPtrs) {
10548 RTArgs.BasePointersArray = ConstantPointerNull::get(T: VoidPtrPtrTy);
10549 RTArgs.PointersArray = ConstantPointerNull::get(T: VoidPtrPtrTy);
10550 RTArgs.SizesArray = ConstantPointerNull::get(T: Int64PtrTy);
10551 RTArgs.MapTypesArray = ConstantPointerNull::get(T: Int64PtrTy);
10552 RTArgs.MapNamesArray = ConstantPointerNull::get(T: VoidPtrPtrTy);
10553 RTArgs.MappersArray = ConstantPointerNull::get(T: VoidPtrPtrTy);
10554 return;
10555 }
10556
10557 RTArgs.BasePointersArray = Builder.CreateConstInBoundsGEP2_32(
10558 Ty: ArrayType::get(ElementType: VoidPtrTy, NumElements: Info.NumberOfPtrs),
10559 Ptr: Info.RTArgs.BasePointersArray,
10560 /*Idx0=*/0, /*Idx1=*/0);
10561 RTArgs.PointersArray = Builder.CreateConstInBoundsGEP2_32(
10562 Ty: ArrayType::get(ElementType: VoidPtrTy, NumElements: Info.NumberOfPtrs), Ptr: Info.RTArgs.PointersArray,
10563 /*Idx0=*/0,
10564 /*Idx1=*/0);
10565 RTArgs.SizesArray = Builder.CreateConstInBoundsGEP2_32(
10566 Ty: ArrayType::get(ElementType: Int64Ty, NumElements: Info.NumberOfPtrs), Ptr: Info.RTArgs.SizesArray,
10567 /*Idx0=*/0, /*Idx1=*/0);
10568 RTArgs.MapTypesArray = Builder.CreateConstInBoundsGEP2_32(
10569 Ty: ArrayType::get(ElementType: Int64Ty, NumElements: Info.NumberOfPtrs),
10570 Ptr: ForEndCall && Info.RTArgs.MapTypesArrayEnd ? Info.RTArgs.MapTypesArrayEnd
10571 : Info.RTArgs.MapTypesArray,
10572 /*Idx0=*/0,
10573 /*Idx1=*/0);
10574
10575 // Only emit the mapper information arrays if debug information is
10576 // requested.
10577 if (!Info.EmitDebug)
10578 RTArgs.MapNamesArray = ConstantPointerNull::get(T: VoidPtrPtrTy);
10579 else
10580 RTArgs.MapNamesArray = Builder.CreateConstInBoundsGEP2_32(
10581 Ty: ArrayType::get(ElementType: VoidPtrTy, NumElements: Info.NumberOfPtrs), Ptr: Info.RTArgs.MapNamesArray,
10582 /*Idx0=*/0,
10583 /*Idx1=*/0);
10584 // If there is no user-defined mapper, set the mapper array to nullptr to
10585 // avoid an unnecessary data privatization
10586 if (!Info.HasMapper)
10587 RTArgs.MappersArray = ConstantPointerNull::get(T: VoidPtrPtrTy);
10588 else
10589 RTArgs.MappersArray =
10590 Builder.CreatePointerCast(V: Info.RTArgs.MappersArray, DestTy: VoidPtrPtrTy);
10591}
10592
10593void OpenMPIRBuilder::emitNonContiguousDescriptor(InsertPointTy AllocaIP,
10594 InsertPointTy CodeGenIP,
10595 MapInfosTy &CombinedInfo,
10596 TargetDataInfo &Info) {
10597 MapInfosTy::StructNonContiguousInfo &NonContigInfo =
10598 CombinedInfo.NonContigInfo;
10599
10600 // Build an array of struct descriptor_dim and then assign it to
10601 // offload_args.
10602 //
10603 // struct descriptor_dim {
10604 // uint64_t offset;
10605 // uint64_t count;
10606 // uint64_t stride
10607 // };
10608 Type *Int64Ty = Builder.getInt64Ty();
10609 StructType *DimTy = StructType::create(
10610 Context&: M.getContext(), Elements: ArrayRef<Type *>({Int64Ty, Int64Ty, Int64Ty}),
10611 Name: "struct.descriptor_dim");
10612
10613 enum { OffsetFD = 0, CountFD, StrideFD };
10614 // We need two index variable here since the size of "Dims" is the same as
10615 // the size of Components, however, the size of offset, count, and stride is
10616 // equal to the size of base declaration that is non-contiguous.
10617 for (unsigned I = 0, L = 0, E = NonContigInfo.Dims.size(); I < E; ++I) {
10618 // Skip emitting ir if dimension size is 1 since it cannot be
10619 // non-contiguous.
10620 if (NonContigInfo.Dims[I] == 1)
10621 continue;
10622 Builder.restoreIP(IP: AllocaIP);
10623 ArrayType *ArrayTy = ArrayType::get(ElementType: DimTy, NumElements: NonContigInfo.Dims[I]);
10624 AllocaInst *DimsAddr =
10625 Builder.CreateAlloca(Ty: ArrayTy, /* ArraySize = */ nullptr, Name: "dims");
10626 Info.RTArgs.NonContiguousDescriptors.push_back(Elt: {.MapIndex: I, .Storage: DimsAddr});
10627 Builder.restoreIP(IP: CodeGenIP);
10628 for (unsigned II = 0, EE = NonContigInfo.Dims[I]; II < EE; ++II) {
10629 unsigned RevIdx = EE - II - 1;
10630 Value *DimsLVal = Builder.CreateInBoundsGEP(
10631 Ty: ArrayTy, Ptr: DimsAddr, IdxList: {Builder.getInt64(C: 0), Builder.getInt64(C: II)});
10632 // Offset
10633 Value *OffsetLVal = Builder.CreateStructGEP(Ty: DimTy, Ptr: DimsLVal, Idx: OffsetFD);
10634 Builder.CreateAlignedStore(
10635 Val: NonContigInfo.Offsets[L][RevIdx], Ptr: OffsetLVal,
10636 Align: M.getDataLayout().getPrefTypeAlign(Ty: OffsetLVal->getType()));
10637 // Count
10638 Value *CountLVal = Builder.CreateStructGEP(Ty: DimTy, Ptr: DimsLVal, Idx: CountFD);
10639 Builder.CreateAlignedStore(
10640 Val: NonContigInfo.Counts[L][RevIdx], Ptr: CountLVal,
10641 Align: M.getDataLayout().getPrefTypeAlign(Ty: CountLVal->getType()));
10642 // Stride
10643 Value *StrideLVal = Builder.CreateStructGEP(Ty: DimTy, Ptr: DimsLVal, Idx: StrideFD);
10644 Builder.CreateAlignedStore(
10645 Val: NonContigInfo.Strides[L][RevIdx], Ptr: StrideLVal,
10646 Align: M.getDataLayout().getPrefTypeAlign(Ty: CountLVal->getType()));
10647 }
10648 // args[I] = &dims
10649 Builder.restoreIP(IP: CodeGenIP);
10650 Value *DAddr = Builder.CreatePointerBitCastOrAddrSpaceCast(
10651 V: DimsAddr, DestTy: Builder.getPtrTy());
10652 Value *P = Builder.CreateConstInBoundsGEP2_32(
10653 Ty: ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: Info.NumberOfPtrs),
10654 Ptr: Info.RTArgs.PointersArray, Idx0: 0, Idx1: I);
10655 Builder.CreateAlignedStore(
10656 Val: DAddr, Ptr: P, Align: M.getDataLayout().getPrefTypeAlign(Ty: Builder.getPtrTy()));
10657 ++L;
10658 }
10659}
10660
10661void OpenMPIRBuilder::emitUDMapperArrayInitOrDel(
10662 Function *MapperFn, Value *MapperHandle, Value *Base, Value *Begin,
10663 Value *Size, Value *MapType, Value *MapName, TypeSize ElementSize,
10664 BasicBlock *ExitBB, bool IsInit) {
10665 StringRef Prefix = IsInit ? ".init" : ".del";
10666
10667 // Evaluate if this is an array section.
10668 BasicBlock *BodyBB = BasicBlock::Create(
10669 Context&: M.getContext(), Name: createPlatformSpecificName(Parts: {"omp.array", Prefix}));
10670 Value *IsArray =
10671 Builder.CreateICmpSGT(LHS: Size, RHS: Builder.getInt64(C: 1), Name: "omp.arrayinit.isarray");
10672 Value *DeleteBit = Builder.CreateAnd(
10673 LHS: MapType,
10674 RHS: Builder.getInt64(
10675 C: static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10676 OpenMPOffloadMappingFlags::OMP_MAP_DELETE)));
10677 Value *DeleteCond;
10678 Value *Cond;
10679 if (IsInit) {
10680 // base != begin?
10681 Value *BaseIsBegin = Builder.CreateICmpNE(LHS: Base, RHS: Begin);
10682 Cond = Builder.CreateOr(LHS: IsArray, RHS: BaseIsBegin);
10683 DeleteCond = Builder.CreateIsNull(
10684 Arg: DeleteBit,
10685 Name: createPlatformSpecificName(Parts: {"omp.array", Prefix, ".delete"}));
10686 } else {
10687 Cond = IsArray;
10688 DeleteCond = Builder.CreateIsNotNull(
10689 Arg: DeleteBit,
10690 Name: createPlatformSpecificName(Parts: {"omp.array", Prefix, ".delete"}));
10691 }
10692 Cond = Builder.CreateAnd(LHS: Cond, RHS: DeleteCond);
10693 Builder.CreateCondBr(Cond, True: BodyBB, False: ExitBB);
10694
10695 emitBlock(BB: BodyBB, CurFn: MapperFn);
10696 // Get the array size by multiplying element size and element number (i.e., \p
10697 // Size).
10698 Value *ArraySize = Builder.CreateNUWMul(LHS: Size, RHS: Builder.getInt64(C: ElementSize));
10699 // Remove OMP_MAP_TO and OMP_MAP_FROM from the map type, so that it achieves
10700 // memory allocation/deletion purpose only.
10701 Value *MapTypeArg = Builder.CreateAnd(
10702 LHS: MapType,
10703 RHS: Builder.getInt64(
10704 C: ~static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10705 OpenMPOffloadMappingFlags::OMP_MAP_TO |
10706 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
10707 MapTypeArg = Builder.CreateOr(
10708 LHS: MapTypeArg,
10709 RHS: Builder.getInt64(
10710 C: static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10711 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT)));
10712
10713 // Call the runtime API __tgt_push_mapper_component to fill up the runtime
10714 // data structure.
10715 Value *OffloadingArgs[] = {MapperHandle, Base, Begin,
10716 ArraySize, MapTypeArg, MapName};
10717 createRuntimeFunctionCall(
10718 Callee: getOrCreateRuntimeFunction(M, FnID: OMPRTL___tgt_push_mapper_component),
10719 Args: OffloadingArgs);
10720}
10721
10722Expected<Function *> OpenMPIRBuilder::emitUserDefinedMapper(
10723 function_ref<MapInfosOrErrorTy(InsertPointTy CodeGenIP, llvm::Value *PtrPHI,
10724 llvm::Value *BeginArg)>
10725 GenMapInfoCB,
10726 Type *ElemTy, StringRef FuncName, CustomMapperCallbackTy CustomMapperCB,
10727 bool PreserveMemberOfFlags, bool PropagatePresentToPointeeStorage,
10728 bool PropagatePresentToNonPointeeStorage) {
10729 SmallVector<Type *> Params;
10730 Params.emplace_back(Args: Builder.getPtrTy());
10731 Params.emplace_back(Args: Builder.getPtrTy());
10732 Params.emplace_back(Args: Builder.getPtrTy());
10733 Params.emplace_back(Args: Builder.getInt64Ty());
10734 Params.emplace_back(Args: Builder.getInt64Ty());
10735 Params.emplace_back(Args: Builder.getPtrTy());
10736
10737 auto *FnTy =
10738 FunctionType::get(Result: Builder.getVoidTy(), Params, /* IsVarArg */ isVarArg: false);
10739
10740 Function *MapperFn =
10741 Function::Create(Ty: FnTy, Linkage: GlobalValue::InternalLinkage, N: FuncName, M);
10742 MapperFn->addFnAttr(Kind: Attribute::NoInline);
10743 MapperFn->addFnAttr(Kind: Attribute::NoUnwind);
10744 MapperFn->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
10745 MapperFn->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
10746 MapperFn->addParamAttr(ArgNo: 2, Kind: Attribute::NoUndef);
10747 MapperFn->addParamAttr(ArgNo: 3, Kind: Attribute::NoUndef);
10748 MapperFn->addParamAttr(ArgNo: 4, Kind: Attribute::NoUndef);
10749 MapperFn->addParamAttr(ArgNo: 5, Kind: Attribute::NoUndef);
10750
10751 // Start the mapper function code generation.
10752 BasicBlock *EntryBB = BasicBlock::Create(Context&: M.getContext(), Name: "entry", Parent: MapperFn);
10753 IRBuilder<>::InsertPointGuard IPG(Builder);
10754 Builder.SetInsertPoint(EntryBB);
10755 Builder.SetCurrentDebugLocation(llvm::DebugLoc());
10756
10757 Value *MapperHandle = MapperFn->getArg(i: 0);
10758 Value *BaseIn = MapperFn->getArg(i: 1);
10759 Value *BeginIn = MapperFn->getArg(i: 2);
10760 Value *Size = MapperFn->getArg(i: 3);
10761 Value *MapType = MapperFn->getArg(i: 4);
10762 Value *MapName = MapperFn->getArg(i: 5);
10763
10764 // Compute the starting and end addresses of array elements.
10765 // Prepare common arguments for array initiation and deletion.
10766 // Convert the size in bytes into the number of array elements.
10767 TypeSize ElementSize = M.getDataLayout().getTypeStoreSize(Ty: ElemTy);
10768 Size = Builder.CreateExactUDiv(LHS: Size, RHS: Builder.getInt64(C: ElementSize));
10769 Value *PtrBegin = BeginIn;
10770 Value *PtrEnd = Builder.CreateGEP(Ty: ElemTy, Ptr: PtrBegin, IdxList: Size);
10771
10772 // Emit array initiation if this is an array section and \p MapType indicates
10773 // that memory allocation is required.
10774 BasicBlock *HeadBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp.arraymap.head");
10775 emitUDMapperArrayInitOrDel(MapperFn, MapperHandle, Base: BaseIn, Begin: BeginIn, Size,
10776 MapType, MapName, ElementSize, ExitBB: HeadBB,
10777 /*IsInit=*/true);
10778
10779 // Emit a for loop to iterate through SizeArg of elements and map all of them.
10780
10781 // Emit the loop header block.
10782 emitBlock(BB: HeadBB, CurFn: MapperFn);
10783 BasicBlock *BodyBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp.arraymap.body");
10784 BasicBlock *DoneBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp.done");
10785 // Evaluate whether the initial condition is satisfied.
10786 Value *IsEmpty =
10787 Builder.CreateICmpEQ(LHS: PtrBegin, RHS: PtrEnd, Name: "omp.arraymap.isempty");
10788 Builder.CreateCondBr(Cond: IsEmpty, True: DoneBB, False: BodyBB);
10789
10790 // Emit the loop body block.
10791 emitBlock(BB: BodyBB, CurFn: MapperFn);
10792 BasicBlock *LastBB = BodyBB;
10793 PHINode *PtrPHI =
10794 Builder.CreatePHI(Ty: PtrBegin->getType(), NumReservedValues: 2, Name: "omp.arraymap.ptrcurrent");
10795 PtrPHI->addIncoming(V: PtrBegin, BB: HeadBB);
10796
10797 // Get map clause information. Fill up the arrays with all mapped variables.
10798 MapInfosOrErrorTy Info = GenMapInfoCB(Builder.saveIP(), PtrPHI, BeginIn);
10799 if (!Info)
10800 return Info.takeError();
10801
10802 // Number of components already in the mapper handle, shifted into the
10803 // MEMBER_OF field; assigned below, once a conditional leading parent (if
10804 // any) has been pushed, and read by pushComponent through the reference
10805 // capture.
10806 Value *ShiftedPreviousSize = nullptr;
10807
10808 // Emits a single mapper sub-component: computes the MEMBER_OF-adjusted map
10809 // type, applies map-type-modifier propagation, and pushes it either via a
10810 // child mapper function or __tgt_push_mapper_component. Returns the last
10811 // basic block emitted, so callers can update the outer per-array-element
10812 // loop's PHI incoming block.
10813 //
10814 // Add MEMBER_OF (ShiftedPreviousSize) to group this sub-map with the
10815 // current array element (N = __tgt_mapper_num_components() at loop body
10816 // start).
10817 //
10818 // Example 1:
10819 // struct S { int x; int *p; };
10820 //
10821 // mapper: #pragma omp declare mapper(id: S s) map(s.x, s.p[0:10])
10822 // use: S arr[2]; ... map(arr)
10823 // entries per element:
10824 //
10825 // &arr[i], &arr[i].x, sizeof(int), MEMBER_OF(N)|TO|FROM
10826 // &arr[i].p[0], &arr[i].p[0], 10*sizeof(int), TO|FROM (*)
10827 // &arr[i].p, &arr[i].p[0], sizeof(int*), ATTACH (**)
10828 //
10829 // Example 2:
10830 // struct S1 { int x; int y; };
10831 // struct S2 { int z; S1 *s1p; };
10832 //
10833 // mapper: #pragma omp declare mapper(S2 s2) map(s2.z, s2.s1p->x,
10834 // s2.s1p->y)
10835 // use: S2 arr[2]; ... map(arr)
10836 // entries per element:
10837 //
10838 // &arr[i], &arr[i].z, sizeof(int), MEMBER_OF(N)|TO|FROM
10839 // &arr[i].s1p[0], &arr[i].s1p->x, sizeof(s1p->x..y), ALLOC (*)
10840 // &arr[i].s1p[0], &arr[i].s1p->x, 4, MEMBER_OF(N+2)|TO|FROM (*)(***)
10841 // &arr[i].s1p[0], &arr[i].s1p->y, 4, MEMBER_OF(N+2)|TO|FROM (*)(***)
10842 // &arr[i].s1p, &arr[i].s1p->x, sizeof(ptr), ATTACH (**)
10843 //
10844 // x/y carry inner MEMBER_OF(2)
10845 // which is shifted by N to become MEMBER_OF(N+2).
10846 //
10847 // HasAttachPtr is set on all of the s1p entries except the ATTACH one:
10848 // the combined ALLOC entry for the s1p->x..y block, and the individual
10849 // x/y entries that are MEMBER_OF that block, all describe storage
10850 // reached through the attach ptr arr[i].s1p.
10851 //
10852 // Entries of the following kinds do NOT receive a new outer MEMBER_OF
10853 // linking them to the parent struct:
10854 //
10855 // * (*) Entries with HasAttachPtr: they represent pointee data that
10856 // occupies a different storage block than the struct being mapped, so
10857 // they are not a member of it. They may still be MEMBER_OF an entry
10858 // within that pointee block, in which case those pre-existing bits are
10859 // shifted -- see (***).
10860 // * (**) ATTACH entries: they are not a member of anything — they just
10861 // link a ptr to its ptee.
10862 // * All entries when PreserveMemberOfFlags is set (the Flang/MLIR path):
10863 // its pre-shaped entries already carry their final MEMBER_OF bits.
10864 // TODO: set HasAttachPtr from Flang for entries whose storage is the
10865 // pointee's (e.g. s%p(0:10)) and drop PreserveMemberOfFlags in favor of
10866 // it.
10867 //
10868 // (***) If such an entry already has its own MEMBER_OF bits (e.g. the
10869 // s1p->x/y entries above), those bits are still shifted by N.
10870 auto pushComponent = [&](Value *CurBaseArg, Value *CurBeginArg,
10871 Value *CurSizeArg, Value *CurNameArg,
10872 uint64_t RawType, bool HasAttachPtrFlag,
10873 Function *ChildMapperFn) -> BasicBlock * {
10874 Value *OriMapType = Builder.getInt64(C: RawType);
10875 constexpr uint64_t MemberOfMask =
10876 static_cast<uint64_t>(OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);
10877 constexpr uint64_t AttachBit =
10878 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10879 OpenMPOffloadMappingFlags::OMP_MAP_ATTACH);
10880 constexpr uint64_t PtrAndObjBit =
10881 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10882 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ);
10883
10884 Value *MemberMapType;
10885 if (PreserveMemberOfFlags || (RawType & AttachBit) || HasAttachPtrFlag) {
10886 if (RawType & MemberOfMask)
10887 MemberMapType = Builder.CreateNUWAdd(LHS: OriMapType, RHS: ShiftedPreviousSize);
10888 else
10889 MemberMapType = OriMapType;
10890 } else {
10891 MemberMapType = Builder.CreateNUWAdd(LHS: OriMapType, RHS: ShiftedPreviousSize);
10892 }
10893
10894 // Combine the map type inherited from user-defined mapper with that
10895 // specified in the program. According to the OMP_MAP_TO and OMP_MAP_FROM
10896 // bits of the \a MapType, which is the input argument of the mapper
10897 // function, the following code will set the OMP_MAP_TO and OMP_MAP_FROM
10898 // bits of MemberMapType.
10899 // [OpenMP 5.0], 1.2.6. map-type decay.
10900 // | alloc | to | from | tofrom | release | delete
10901 // ----------------------------------------------------------
10902 // alloc | alloc | alloc | alloc | alloc | release | delete
10903 // to | alloc | to | alloc | to | release | delete
10904 // from | alloc | alloc | from | from | release | delete
10905 // tofrom | alloc | to | from | tofrom | release | delete
10906 Value *LeftToFrom = Builder.CreateAnd(
10907 LHS: MapType,
10908 RHS: Builder.getInt64(
10909 C: static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10910 OpenMPOffloadMappingFlags::OMP_MAP_TO |
10911 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
10912 BasicBlock *AllocBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp.type.alloc");
10913 BasicBlock *AllocElseBB =
10914 BasicBlock::Create(Context&: M.getContext(), Name: "omp.type.alloc.else");
10915 BasicBlock *ToBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp.type.to");
10916 BasicBlock *ToElseBB =
10917 BasicBlock::Create(Context&: M.getContext(), Name: "omp.type.to.else");
10918 BasicBlock *FromBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp.type.from");
10919 BasicBlock *EndBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp.type.end");
10920 Value *IsAlloc = Builder.CreateIsNull(Arg: LeftToFrom);
10921 Builder.CreateCondBr(Cond: IsAlloc, True: AllocBB, False: AllocElseBB);
10922 // In case of alloc, clear OMP_MAP_TO and OMP_MAP_FROM.
10923 emitBlock(BB: AllocBB, CurFn: MapperFn);
10924 Value *AllocMapType = Builder.CreateAnd(
10925 LHS: MemberMapType,
10926 RHS: Builder.getInt64(
10927 C: ~static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10928 OpenMPOffloadMappingFlags::OMP_MAP_TO |
10929 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
10930 Builder.CreateBr(Dest: EndBB);
10931 emitBlock(BB: AllocElseBB, CurFn: MapperFn);
10932 Value *IsTo = Builder.CreateICmpEQ(
10933 LHS: LeftToFrom,
10934 RHS: Builder.getInt64(
10935 C: static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10936 OpenMPOffloadMappingFlags::OMP_MAP_TO)));
10937 Builder.CreateCondBr(Cond: IsTo, True: ToBB, False: ToElseBB);
10938 // In case of to, clear OMP_MAP_FROM.
10939 emitBlock(BB: ToBB, CurFn: MapperFn);
10940 Value *ToMapType = Builder.CreateAnd(
10941 LHS: MemberMapType,
10942 RHS: Builder.getInt64(
10943 C: ~static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10944 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
10945 Builder.CreateBr(Dest: EndBB);
10946 emitBlock(BB: ToElseBB, CurFn: MapperFn);
10947 Value *IsFrom = Builder.CreateICmpEQ(
10948 LHS: LeftToFrom,
10949 RHS: Builder.getInt64(
10950 C: static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10951 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
10952 Builder.CreateCondBr(Cond: IsFrom, True: FromBB, False: EndBB);
10953 // In case of from, clear OMP_MAP_TO.
10954 emitBlock(BB: FromBB, CurFn: MapperFn);
10955 Value *FromMapType = Builder.CreateAnd(
10956 LHS: MemberMapType,
10957 RHS: Builder.getInt64(
10958 C: ~static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10959 OpenMPOffloadMappingFlags::OMP_MAP_TO)));
10960 // In case of tofrom, do nothing.
10961 emitBlock(BB: EndBB, CurFn: MapperFn);
10962 PHINode *CurMapType =
10963 Builder.CreatePHI(Ty: Builder.getInt64Ty(), NumReservedValues: 4, Name: "omp.maptype");
10964 CurMapType->addIncoming(V: AllocMapType, BB: AllocBB);
10965 CurMapType->addIncoming(V: ToMapType, BB: ToBB);
10966 CurMapType->addIncoming(V: FromMapType, BB: FromBB);
10967 CurMapType->addIncoming(V: MemberMapType, BB: ToElseBB);
10968
10969 // Propagate map-type-modifying bits from the outer map clause to each map
10970 // inserted by the mapper.
10971 //
10972 // OpenMP 6.0:281:34: The effect of the mapper modifier is to remove the
10973 // list item from the map clause and to apply the clauses specified in the
10974 // declared mapper to the construct on which the map clause appears...
10975 // If any modifier with the map-type-modifying property appears in the map
10976 // clause then the effect is as if that modifier appears in each map clause
10977 // specified in the declared mapper.
10978 //
10979 // Map-type-modifying bits: ALWAYS, DELETE, CLOSE, PRESENT.
10980 //
10981 // ALWAYS/DELETE/CLOSE are propagated to every (non-ATTACH) entry.
10982 //
10983 // PRESENT propagation is controlled independently for entries describing
10984 // pointee storage (HasAttachPtr) and entries describing non-pointee
10985 // storage. This lets frontends select current Clang behavior (pointee only)
10986 // or OpenMP 6.x behavior for mapper propagation (all eligible storage).
10987 // PRESENT is not propagated to ATTACH or PTR_AND_OBJ entries.
10988 //
10989 // Pointee storage is gated on \p PropagatePresentToPointeeStorage (set by
10990 // caller only for OpenMP >= 6.0). Before 6.0 the present modifier is
10991 // treated as not applying to the pointee: the spec committee confirmed the
10992 // divergence between the present "motion" modifier (to/from) and the
10993 // present map-type modifier (map) was unintentional, to be fixed as an
10994 // OpenMP 6.0 erratum, so for 5.2 present is ignored for the pointee for
10995 // both map and to/from.
10996 //
10997 // Non-pointee storage is gated on \p PropagatePresentToNonPointeeStorage:
10998 // PRESENT is ten propagated to the struct's own members (e.g. the s.x,
10999 // s.y of map(present, mapper(id): s)) so that an absent member triggers
11000 // the present-check.
11001 //
11002 // Present cannot be mapped to members with PTR_AND_OBJ, a single combined
11003 // entry allocates the whole struct (including the pointer's storage),
11004 // so propagating PRESENT to it would wrongly require the pointer's
11005 // pointee to be present.
11006 uint64_t ModifierBits =
11007 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
11008 OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS |
11009 OpenMPOffloadMappingFlags::OMP_MAP_DELETE |
11010 OpenMPOffloadMappingFlags::OMP_MAP_CLOSE);
11011 bool IsPresentEligibleStorage = !(RawType & (AttachBit | PtrAndObjBit));
11012 bool PropagatePresent =
11013 IsPresentEligibleStorage &&
11014 (HasAttachPtrFlag ? PropagatePresentToPointeeStorage
11015 : PropagatePresentToNonPointeeStorage);
11016 if (PropagatePresent)
11017 ModifierBits |=
11018 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
11019 OpenMPOffloadMappingFlags::OMP_MAP_PRESENT);
11020 Value *ImportedModifierBits =
11021 Builder.CreateAnd(LHS: MapType, RHS: Builder.getInt64(C: ModifierBits));
11022 Value *CurMapTypeWithModifiers = Builder.CreateOr(
11023 LHS: CurMapType, RHS: ImportedModifierBits, Name: "omp.maptype.with.modifiers");
11024
11025 // ATTACH entries must not receive map-type-modifying bits: ATTACH|ALWAYS is
11026 // reserved for the attach(always) map-type modifier, and other modifier
11027 // bits (DELETE, CLOSE) have no meaning for an ATTACH entry.
11028 Value *FinalMapType =
11029 (RawType & AttachBit) ? CurMapType : CurMapTypeWithModifiers;
11030
11031 Value *OffloadingArgs[] = {MapperHandle, CurBaseArg, CurBeginArg,
11032 CurSizeArg, FinalMapType, CurNameArg};
11033 if (ChildMapperFn) {
11034 // Call the corresponding mapper function.
11035 createRuntimeFunctionCall(Callee: ChildMapperFn, Args: OffloadingArgs)
11036 ->setDoesNotThrow();
11037 } else {
11038 // Call the runtime API __tgt_push_mapper_component to fill up the runtime
11039 // data structure.
11040 createRuntimeFunctionCall(
11041 Callee: getOrCreateRuntimeFunction(M, FnID: OMPRTL___tgt_push_mapper_component),
11042 Args: OffloadingArgs);
11043 }
11044 return EndBB;
11045 };
11046
11047 // A leading parent synthesized for the dynamic segments is pushed only when
11048 // some segment will contribute components. Its own map type carries no
11049 // MEMBER_OF bits, so it does not read ShiftedPreviousSize.
11050 unsigned FirstStaticIdx = 0;
11051 if (Info->LeadingParentCond) {
11052 assert(PreserveMemberOfFlags &&
11053 "conditional leading parent requires pre-shaped MEMBER_OF flags");
11054 assert(!Info->BasePointers.empty() && "no leading parent component");
11055 BasicBlock *PushBB =
11056 BasicBlock::Create(Context&: M.getContext(), Name: "omp.iterparent.push");
11057 BasicBlock *ContBB =
11058 BasicBlock::Create(Context&: M.getContext(), Name: "omp.iterparent.cont");
11059 Builder.CreateCondBr(Cond: Info->LeadingParentCond, True: PushBB, False: ContBB);
11060 emitBlock(BB: PushBB, CurFn: MapperFn);
11061 Value *CurNameArg = Info->Names.size()
11062 ? Info->Names[0]
11063 : Constant::getNullValue(Ty: Builder.getPtrTy());
11064 pushComponent(
11065 Info->BasePointers[0], Info->Pointers[0], Info->Sizes[0], CurNameArg,
11066 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
11067 Info->Types[0]),
11068 Info->HasAttachPtr[0], /*ChildMapperFn=*/nullptr);
11069 Builder.CreateBr(Dest: ContBB);
11070 emitBlock(BB: ContBB, CurFn: MapperFn);
11071 LastBB = ContBB;
11072 FirstStaticIdx = 1;
11073 }
11074
11075 // Call the runtime API __tgt_mapper_num_components to get the number of
11076 // pre-existing components. The compile-time MEMBER_OF indices count the
11077 // leading parent as component 0 whether or not it was pushed; reading the
11078 // count after that push and subtracting one yields the right base in both
11079 // cases (N if pushed, N - 1 if skipped, N being the count at body start).
11080 Value *OffloadingArgs[] = {MapperHandle};
11081 Value *PreviousSize = createRuntimeFunctionCall(
11082 Callee: getOrCreateRuntimeFunction(M, FnID: OMPRTL___tgt_mapper_num_components),
11083 Args: OffloadingArgs);
11084 if (Info->LeadingParentCond)
11085 PreviousSize = Builder.CreateSub(LHS: PreviousSize, RHS: Builder.getInt64(C: 1));
11086 ShiftedPreviousSize =
11087 Builder.CreateShl(LHS: PreviousSize, RHS: Builder.getInt64(C: getFlagMemberOffset()));
11088
11089 // Fill up the runtime mapper handle for all statically-known components.
11090 for (unsigned I = FirstStaticIdx; I < Info->BasePointers.size(); ++I) {
11091 Value *CurNameArg = Info->Names.size()
11092 ? Info->Names[I]
11093 : Constant::getNullValue(Ty: Builder.getPtrTy());
11094 Expected<Function *> ChildMapperFn = CustomMapperCB(I);
11095 if (!ChildMapperFn)
11096 return ChildMapperFn.takeError();
11097 LastBB = pushComponent(
11098 Info->BasePointers[I], Info->Pointers[I], Info->Sizes[I], CurNameArg,
11099 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
11100 Info->Types[I]),
11101 Info->HasAttachPtr[I], *ChildMapperFn);
11102 }
11103
11104 // Fill up the runtime mapper handle for components whose count is only
11105 // known at runtime (produced by an `iterator` modifier on the mapper's own
11106 // map clause): push each one individually, from inside a generated loop,
11107 // via the same __tgt_push_mapper_component call used above.
11108 for (MapInfosTy::DynamicSegment &Seg : Info->DynamicSegments) {
11109 LocationDescription Loc(Builder);
11110 auto BodyGen = [&](InsertPointTy BodyIP, Value *LinearIV) -> Error {
11111 Builder.restoreIP(IP: BodyIP);
11112 Expected<SmallVector<Value *, 3>> EntryOrErr =
11113 Seg.GenEntry(Builder, LinearIV);
11114 if (!EntryOrErr)
11115 return EntryOrErr.takeError();
11116 SmallVector<Value *, 3> &Entry = *EntryOrErr;
11117 pushComponent(
11118 Entry[0], Entry[1], Entry[2],
11119 Constant::getNullValue(Ty: Builder.getPtrTy()),
11120 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
11121 Seg.Type),
11122 Seg.HasAttachPtr, Seg.ChildMapperFn);
11123 return Error::success();
11124 };
11125 InsertPointOrErrorTy AfterIP =
11126 createIteratorLoop(Loc, TripCount: Seg.Count, BodyGen, Name: "mapper.iterator");
11127 if (!AfterIP)
11128 return AfterIP.takeError();
11129 Builder.restoreIP(IP: *AfterIP);
11130 LastBB = Builder.GetInsertBlock();
11131 }
11132
11133 // Update the pointer to point to the next element that needs to be mapped,
11134 // and check whether we have mapped all elements.
11135 Value *PtrNext = Builder.CreateConstGEP1_32(Ty: ElemTy, Ptr: PtrPHI, /*Idx0=*/1,
11136 Name: "omp.arraymap.next");
11137 PtrPHI->addIncoming(V: PtrNext, BB: LastBB);
11138 Value *IsDone = Builder.CreateICmpEQ(LHS: PtrNext, RHS: PtrEnd, Name: "omp.arraymap.isdone");
11139 BasicBlock *ExitBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp.arraymap.exit");
11140 Builder.CreateCondBr(Cond: IsDone, True: ExitBB, False: BodyBB);
11141
11142 emitBlock(BB: ExitBB, CurFn: MapperFn);
11143 // Emit array deletion if this is an array section and \p MapType indicates
11144 // that deletion is required.
11145 emitUDMapperArrayInitOrDel(MapperFn, MapperHandle, Base: BaseIn, Begin: BeginIn, Size,
11146 MapType, MapName, ElementSize, ExitBB: DoneBB,
11147 /*IsInit=*/false);
11148
11149 // Emit the function exit block.
11150 emitBlock(BB: DoneBB, CurFn: MapperFn, /*IsFinished=*/true);
11151
11152 Builder.CreateRetVoid();
11153 return MapperFn;
11154}
11155
11156Error OpenMPIRBuilder::emitOffloadingArrays(
11157 InsertPointTy AllocaIP, InsertPointTy CodeGenIP, MapInfosTy &CombinedInfo,
11158 TargetDataInfo &Info, CustomMapperCallbackTy CustomMapperCB,
11159 bool IsNonContiguous,
11160 function_ref<void(unsigned int, Value *)> DeviceAddrCB) {
11161
11162 // Reset the array information.
11163 Info.clearArrayInfo();
11164 Info.NumberOfPtrs = CombinedInfo.BasePointers.size();
11165
11166 if (Info.NumberOfPtrs == 0)
11167 return Error::success();
11168
11169 Builder.restoreIP(IP: AllocaIP);
11170 // Detect if we have any capture size requiring runtime evaluation of the
11171 // size so that a constant array could be eventually used.
11172 ArrayType *PointerArrayType =
11173 ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: Info.NumberOfPtrs);
11174
11175 Info.RTArgs.BasePointersArray = Builder.CreateAlloca(
11176 Ty: PointerArrayType, /* ArraySize = */ nullptr, Name: ".offload_baseptrs");
11177
11178 Info.RTArgs.PointersArray = Builder.CreateAlloca(
11179 Ty: PointerArrayType, /* ArraySize = */ nullptr, Name: ".offload_ptrs");
11180 AllocaInst *MappersArray = Builder.CreateAlloca(
11181 Ty: PointerArrayType, /* ArraySize = */ nullptr, Name: ".offload_mappers");
11182 Info.RTArgs.MappersArray = MappersArray;
11183
11184 // If we don't have any VLA types or other types that require runtime
11185 // evaluation, we can use a constant array for the map sizes, otherwise we
11186 // need to fill up the arrays as we do for the pointers.
11187 Type *Int64Ty = Builder.getInt64Ty();
11188 SmallVector<Constant *> ConstSizes(CombinedInfo.Sizes.size(),
11189 ConstantInt::get(Ty: Int64Ty, V: 0));
11190 SmallBitVector RuntimeSizes(CombinedInfo.Sizes.size());
11191 for (unsigned I = 0, E = CombinedInfo.Sizes.size(); I < E; ++I) {
11192 bool IsNonContigEntry =
11193 IsNonContiguous &&
11194 (static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
11195 CombinedInfo.Types[I] &
11196 OpenMPOffloadMappingFlags::OMP_MAP_NON_CONTIG) != 0);
11197 // For NON_CONTIG entries, ArgSizes stores the dimension count (number of
11198 // descriptor_dim records), not the byte size.
11199 if (IsNonContigEntry) {
11200 assert(I < CombinedInfo.NonContigInfo.Dims.size() &&
11201 "Index must be in-bounds for NON_CONTIG Dims array");
11202 const uint64_t DimCount = CombinedInfo.NonContigInfo.Dims[I];
11203 assert(DimCount > 0 && "NON_CONTIG DimCount must be > 0");
11204 ConstSizes[I] = ConstantInt::get(Ty: Int64Ty, V: DimCount);
11205 continue;
11206 }
11207 if (auto *CI = dyn_cast<Constant>(Val: CombinedInfo.Sizes[I])) {
11208 if (!isa<ConstantExpr>(Val: CI) && !isa<GlobalValue>(Val: CI)) {
11209 ConstSizes[I] = CI;
11210 continue;
11211 }
11212 }
11213 RuntimeSizes.set(I);
11214 }
11215
11216 if (RuntimeSizes.all()) {
11217 ArrayType *SizeArrayType = ArrayType::get(ElementType: Int64Ty, NumElements: Info.NumberOfPtrs);
11218 Info.RTArgs.SizesArray = Builder.CreateAlloca(
11219 Ty: SizeArrayType, /* ArraySize = */ nullptr, Name: ".offload_sizes");
11220 restoreIPandDebugLoc(Builder, IP: CodeGenIP);
11221 } else {
11222 auto *SizesArrayInit = ConstantArray::get(
11223 T: ArrayType::get(ElementType: Int64Ty, NumElements: ConstSizes.size()), V: ConstSizes);
11224 std::string Name = createPlatformSpecificName(Parts: {"offload_sizes"});
11225 auto *SizesArrayGbl =
11226 new GlobalVariable(M, SizesArrayInit->getType(), /*isConstant=*/true,
11227 GlobalValue::PrivateLinkage, SizesArrayInit, Name);
11228 SizesArrayGbl->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
11229
11230 if (!RuntimeSizes.any()) {
11231 Info.RTArgs.SizesArray = SizesArrayGbl;
11232 } else {
11233 unsigned IndexSize = M.getDataLayout().getIndexSizeInBits(AS: 0);
11234 Align OffloadSizeAlign = M.getDataLayout().getABIIntegerTypeAlignment(BitWidth: 64);
11235 ArrayType *SizeArrayType = ArrayType::get(ElementType: Int64Ty, NumElements: Info.NumberOfPtrs);
11236 AllocaInst *Buffer = Builder.CreateAlloca(
11237 Ty: SizeArrayType, /* ArraySize = */ nullptr, Name: ".offload_sizes");
11238 Buffer->setAlignment(OffloadSizeAlign);
11239 restoreIPandDebugLoc(Builder, IP: CodeGenIP);
11240 Builder.CreateMemCpy(
11241 Dst: Buffer, DstAlign: M.getDataLayout().getPrefTypeAlign(Ty: Buffer->getType()),
11242 Src: SizesArrayGbl, SrcAlign: OffloadSizeAlign,
11243 Size: Builder.getIntN(
11244 N: IndexSize,
11245 C: Buffer->getAllocationSize(DL: M.getDataLayout())->getFixedValue()));
11246
11247 Info.RTArgs.SizesArray = Buffer;
11248 }
11249 restoreIPandDebugLoc(Builder, IP: CodeGenIP);
11250 }
11251
11252 // The map types are always constant so we don't need to generate code to
11253 // fill arrays. Instead, we create an array constant.
11254 SmallVector<uint64_t, 4> Mapping;
11255 for (auto mapFlag : CombinedInfo.Types)
11256 Mapping.push_back(
11257 Elt: static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
11258 mapFlag));
11259 std::string MaptypesName = createPlatformSpecificName(Parts: {"offload_maptypes"});
11260 auto *MapTypesArrayGbl = createOffloadMaptypes(Mappings&: Mapping, VarName: MaptypesName);
11261 Info.RTArgs.MapTypesArray = MapTypesArrayGbl;
11262
11263 // The information types are only built if provided.
11264 if (!CombinedInfo.Names.empty()) {
11265 auto *MapNamesArrayGbl = createOffloadMapnames(
11266 Names&: CombinedInfo.Names, VarName: createPlatformSpecificName(Parts: {"offload_mapnames"}));
11267 Info.RTArgs.MapNamesArray = MapNamesArrayGbl;
11268 Info.EmitDebug = true;
11269 } else {
11270 Info.RTArgs.MapNamesArray =
11271 Constant::getNullValue(Ty: PointerType::getUnqual(C&: Builder.getContext()));
11272 Info.EmitDebug = false;
11273 }
11274
11275 // If there's a present map type modifier, it must not be applied to the end
11276 // of a region, so generate a separate map type array in that case.
11277 if (Info.separateBeginEndCalls()) {
11278 bool EndMapTypesDiffer = false;
11279 for (uint64_t &Type : Mapping) {
11280 if (Type & static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
11281 OpenMPOffloadMappingFlags::OMP_MAP_PRESENT)) {
11282 Type &= ~static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
11283 OpenMPOffloadMappingFlags::OMP_MAP_PRESENT);
11284 EndMapTypesDiffer = true;
11285 }
11286 }
11287 if (EndMapTypesDiffer) {
11288 MapTypesArrayGbl = createOffloadMaptypes(Mappings&: Mapping, VarName: MaptypesName);
11289 Info.RTArgs.MapTypesArrayEnd = MapTypesArrayGbl;
11290 }
11291 }
11292
11293 PointerType *PtrTy = Builder.getPtrTy();
11294 for (unsigned I = 0; I < Info.NumberOfPtrs; ++I) {
11295 Value *BPVal = CombinedInfo.BasePointers[I];
11296 Value *BP = Builder.CreateConstInBoundsGEP2_32(
11297 Ty: ArrayType::get(ElementType: PtrTy, NumElements: Info.NumberOfPtrs), Ptr: Info.RTArgs.BasePointersArray,
11298 Idx0: 0, Idx1: I);
11299 Builder.CreateAlignedStore(Val: BPVal, Ptr: BP,
11300 Align: M.getDataLayout().getPrefTypeAlign(Ty: PtrTy));
11301
11302 if (Info.requiresDevicePointerInfo()) {
11303 if (CombinedInfo.DevicePointers[I] == DeviceInfoTy::Pointer) {
11304 CodeGenIP = Builder.saveIP();
11305 Builder.restoreIP(IP: AllocaIP);
11306 Info.DevicePtrInfoMap[BPVal] = {BP, Builder.CreateAlloca(Ty: PtrTy)};
11307 restoreIPandDebugLoc(Builder, IP: CodeGenIP);
11308 if (DeviceAddrCB)
11309 DeviceAddrCB(I, Info.DevicePtrInfoMap[BPVal].second);
11310 } else if (CombinedInfo.DevicePointers[I] == DeviceInfoTy::Address) {
11311 Info.DevicePtrInfoMap[BPVal] = {BP, BP};
11312 if (DeviceAddrCB)
11313 DeviceAddrCB(I, BP);
11314 }
11315 }
11316
11317 Value *PVal = CombinedInfo.Pointers[I];
11318 Value *P = Builder.CreateConstInBoundsGEP2_32(
11319 Ty: ArrayType::get(ElementType: PtrTy, NumElements: Info.NumberOfPtrs), Ptr: Info.RTArgs.PointersArray, Idx0: 0,
11320 Idx1: I);
11321 // TODO: Check alignment correct.
11322 Builder.CreateAlignedStore(Val: PVal, Ptr: P,
11323 Align: M.getDataLayout().getPrefTypeAlign(Ty: PtrTy));
11324
11325 if (RuntimeSizes.test(Idx: I)) {
11326 Value *S = Builder.CreateConstInBoundsGEP2_32(
11327 Ty: ArrayType::get(ElementType: Int64Ty, NumElements: Info.NumberOfPtrs), Ptr: Info.RTArgs.SizesArray,
11328 /*Idx0=*/0,
11329 /*Idx1=*/I);
11330 Builder.CreateAlignedStore(Val: Builder.CreateIntCast(V: CombinedInfo.Sizes[I],
11331 DestTy: Int64Ty,
11332 /*isSigned=*/true),
11333 Ptr: S, Align: M.getDataLayout().getPrefTypeAlign(Ty: PtrTy));
11334 }
11335 // Fill up the mapper array.
11336 unsigned IndexSize = M.getDataLayout().getIndexSizeInBits(AS: 0);
11337 Value *MFunc = ConstantPointerNull::get(T: PtrTy);
11338
11339 auto CustomMFunc = CustomMapperCB(I);
11340 if (!CustomMFunc)
11341 return CustomMFunc.takeError();
11342 if (*CustomMFunc)
11343 MFunc = Builder.CreatePointerCast(V: *CustomMFunc, DestTy: PtrTy);
11344
11345 Value *MAddr = Builder.CreateInBoundsGEP(
11346 Ty: PointerArrayType, Ptr: MappersArray,
11347 IdxList: {Builder.getIntN(N: IndexSize, C: 0), Builder.getIntN(N: IndexSize, C: I)});
11348 Builder.CreateAlignedStore(
11349 Val: MFunc, Ptr: MAddr, Align: M.getDataLayout().getPrefTypeAlign(Ty: MAddr->getType()));
11350 }
11351
11352 if (!IsNonContiguous || CombinedInfo.NonContigInfo.Offsets.empty() ||
11353 Info.NumberOfPtrs == 0)
11354 return Error::success();
11355 emitNonContiguousDescriptor(AllocaIP, CodeGenIP, CombinedInfo, Info);
11356 return Error::success();
11357}
11358
11359void OpenMPIRBuilder::emitBranch(BasicBlock *Target) {
11360 BasicBlock *CurBB = Builder.GetInsertBlock();
11361
11362 if (!CurBB || CurBB->hasTerminator()) {
11363 // If there is no insert point or the previous block is already
11364 // terminated, don't touch it.
11365 } else {
11366 // Otherwise, create a fall-through branch.
11367 Builder.CreateBr(Dest: Target);
11368 }
11369
11370 Builder.ClearInsertionPoint();
11371}
11372
11373void OpenMPIRBuilder::emitBlock(BasicBlock *BB, Function *CurFn,
11374 bool IsFinished) {
11375 BasicBlock *CurBB = Builder.GetInsertBlock();
11376
11377 // Fall out of the current block (if necessary).
11378 emitBranch(Target: BB);
11379
11380 if (IsFinished && BB->use_empty()) {
11381 BB->eraseFromParent();
11382 return;
11383 }
11384
11385 // Place the block after the current block, if possible, or else at
11386 // the end of the function.
11387 if (CurBB && CurBB->getParent())
11388 CurFn->insert(Position: std::next(x: CurBB->getIterator()), BB);
11389 else
11390 CurFn->insert(Position: CurFn->end(), BB);
11391 Builder.SetInsertPoint(BB);
11392}
11393
11394Error OpenMPIRBuilder::emitIfClause(Value *Cond, BodyGenCallbackTy ThenGen,
11395 BodyGenCallbackTy ElseGen,
11396 InsertPointTy AllocaIP,
11397 ArrayRef<BasicBlock *> DeallocBlocks) {
11398 // If the condition constant folds and can be elided, try to avoid emitting
11399 // the condition and the dead arm of the if/else.
11400 if (auto *CI = dyn_cast<ConstantInt>(Val: Cond)) {
11401 auto CondConstant = CI->getSExtValue();
11402 if (CondConstant)
11403 return ThenGen(AllocaIP, Builder.saveIP(), DeallocBlocks);
11404
11405 return ElseGen(AllocaIP, Builder.saveIP(), DeallocBlocks);
11406 }
11407
11408 Function *CurFn = Builder.GetInsertBlock()->getParent();
11409
11410 // Otherwise, the condition did not fold, or we couldn't elide it. Just
11411 // emit the conditional branch.
11412 BasicBlock *ThenBlock = BasicBlock::Create(Context&: M.getContext(), Name: "omp_if.then");
11413 BasicBlock *ElseBlock = BasicBlock::Create(Context&: M.getContext(), Name: "omp_if.else");
11414 BasicBlock *ContBlock = BasicBlock::Create(Context&: M.getContext(), Name: "omp_if.end");
11415 Builder.CreateCondBr(Cond, True: ThenBlock, False: ElseBlock);
11416 // Emit the 'then' code.
11417 emitBlock(BB: ThenBlock, CurFn);
11418 if (Error Err = ThenGen(AllocaIP, Builder.saveIP(), DeallocBlocks)) {
11419 // Let the function own the blocks not emitted yet so they are freed.
11420 ElseBlock->insertInto(Parent: CurFn);
11421 ContBlock->insertInto(Parent: CurFn);
11422 return Err;
11423 }
11424 emitBranch(Target: ContBlock);
11425 // Emit the 'else' code if present.
11426 // There is no need to emit line number for unconditional branch.
11427 emitBlock(BB: ElseBlock, CurFn);
11428 if (Error Err = ElseGen(AllocaIP, Builder.saveIP(), DeallocBlocks)) {
11429 ContBlock->insertInto(Parent: CurFn);
11430 return Err;
11431 }
11432 // There is no need to emit line number for unconditional branch.
11433 emitBranch(Target: ContBlock);
11434 // Emit the continuation block for code after the if.
11435 emitBlock(BB: ContBlock, CurFn, /*IsFinished=*/true);
11436 return Error::success();
11437}
11438
11439bool OpenMPIRBuilder::checkAndEmitFlushAfterAtomic(
11440 const LocationDescription &Loc, llvm::AtomicOrdering AO, AtomicKind AK) {
11441 assert(!(AO == AtomicOrdering::NotAtomic ||
11442 AO == llvm::AtomicOrdering::Unordered) &&
11443 "Unexpected Atomic Ordering.");
11444
11445 bool Flush = false;
11446 llvm::AtomicOrdering FlushAO = AtomicOrdering::Monotonic;
11447
11448 switch (AK) {
11449 case Read:
11450 if (AO == AtomicOrdering::Acquire || AO == AtomicOrdering::AcquireRelease ||
11451 AO == AtomicOrdering::SequentiallyConsistent) {
11452 FlushAO = AtomicOrdering::Acquire;
11453 Flush = true;
11454 }
11455 break;
11456 case Write:
11457 case Compare:
11458 case Update:
11459 if (AO == AtomicOrdering::Release || AO == AtomicOrdering::AcquireRelease ||
11460 AO == AtomicOrdering::SequentiallyConsistent) {
11461 FlushAO = AtomicOrdering::Release;
11462 Flush = true;
11463 }
11464 break;
11465 case Capture:
11466 switch (AO) {
11467 case AtomicOrdering::Acquire:
11468 FlushAO = AtomicOrdering::Acquire;
11469 Flush = true;
11470 break;
11471 case AtomicOrdering::Release:
11472 FlushAO = AtomicOrdering::Release;
11473 Flush = true;
11474 break;
11475 case AtomicOrdering::AcquireRelease:
11476 case AtomicOrdering::SequentiallyConsistent:
11477 FlushAO = AtomicOrdering::AcquireRelease;
11478 Flush = true;
11479 break;
11480 default:
11481 // do nothing - leave silently.
11482 break;
11483 }
11484 }
11485
11486 if (Flush) {
11487 // Currently Flush RT call still doesn't take memory_ordering, so for when
11488 // that happens, this tries to do the resolution of which atomic ordering
11489 // to use with but issue the flush call
11490 // TODO: pass `FlushAO` after memory ordering support is added
11491 (void)FlushAO;
11492 emitFlush(Loc);
11493 }
11494
11495 // for AO == AtomicOrdering::Monotonic and all other case combinations
11496 // do nothing
11497 return Flush;
11498}
11499
11500OpenMPIRBuilder::InsertPointTy
11501OpenMPIRBuilder::createAtomicRead(const LocationDescription &Loc,
11502 AtomicOpValue &X, AtomicOpValue &V,
11503 AtomicOrdering AO, InsertPointTy AllocaIP) {
11504 if (!updateToLocation(Loc))
11505 return Loc.IP;
11506
11507 assert(X.Var->getType()->isPointerTy() &&
11508 "OMP Atomic expects a pointer to target memory");
11509 Type *XElemTy = X.ElemTy;
11510 assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||
11511 XElemTy->isPointerTy() || XElemTy->isStructTy()) &&
11512 "OMP atomic read expected a scalar type");
11513
11514 Value *XRead = nullptr;
11515
11516 if (XElemTy->isIntegerTy()) {
11517 LoadInst *XLD =
11518 Builder.CreateLoad(Ty: XElemTy, Ptr: X.Var, isVolatile: X.IsVolatile, Name: "omp.atomic.read");
11519 XLD->setAtomic(Ordering: AO);
11520 XRead = cast<Value>(Val: XLD);
11521 } else if (XElemTy->isStructTy()) {
11522 // FIXME: Add checks to ensure __atomic_load is emitted iff the
11523 // target does not support `atomicrmw` of the size of the struct
11524 LoadInst *OldVal = Builder.CreateLoad(Ty: XElemTy, Ptr: X.Var, Name: "omp.atomic.read");
11525 OldVal->setAtomic(Ordering: AO);
11526 const DataLayout &DL = OldVal->getDataLayout();
11527 unsigned LoadSize = DL.getTypeStoreSize(Ty: XElemTy);
11528 OpenMPIRBuilder::AtomicInfo atomicInfo(
11529 &Builder, XElemTy, LoadSize * 8, LoadSize * 8, OldVal->getAlign(),
11530 OldVal->getAlign(), true /* UseLibcall */, AllocaIP, X.Var);
11531 auto AtomicLoadRes = atomicInfo.EmitAtomicLoadLibcall(AO);
11532 XRead = AtomicLoadRes.first;
11533 OldVal->eraseFromParent();
11534 } else {
11535 // We need to perform atomic op as integer
11536 IntegerType *IntCastTy =
11537 IntegerType::get(C&: M.getContext(), NumBits: XElemTy->getScalarSizeInBits());
11538 LoadInst *XLoad =
11539 Builder.CreateLoad(Ty: IntCastTy, Ptr: X.Var, isVolatile: X.IsVolatile, Name: "omp.atomic.load");
11540 XLoad->setAtomic(Ordering: AO);
11541 if (XElemTy->isFloatingPointTy()) {
11542 XRead = Builder.CreateBitCast(V: XLoad, DestTy: XElemTy, Name: "atomic.flt.cast");
11543 } else {
11544 XRead = Builder.CreateIntToPtr(V: XLoad, DestTy: XElemTy, Name: "atomic.ptr.cast");
11545 }
11546 }
11547 checkAndEmitFlushAfterAtomic(Loc, AO, AK: AtomicKind::Read);
11548 Builder.CreateStore(Val: XRead, Ptr: V.Var, isVolatile: V.IsVolatile);
11549 return Builder.saveIP();
11550}
11551
11552OpenMPIRBuilder::InsertPointTy
11553OpenMPIRBuilder::createAtomicWrite(const LocationDescription &Loc,
11554 AtomicOpValue &X, Value *Expr,
11555 AtomicOrdering AO, InsertPointTy AllocaIP) {
11556 if (!updateToLocation(Loc))
11557 return Loc.IP;
11558
11559 assert(X.Var->getType()->isPointerTy() &&
11560 "OMP Atomic expects a pointer to target memory");
11561 Type *XElemTy = X.ElemTy;
11562 assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||
11563 XElemTy->isPointerTy() || XElemTy->isStructTy()) &&
11564 "OMP atomic write expected a scalar type");
11565
11566 if (XElemTy->isIntegerTy()) {
11567 StoreInst *XSt = Builder.CreateStore(Val: Expr, Ptr: X.Var, isVolatile: X.IsVolatile);
11568 XSt->setAtomic(Ordering: AO);
11569 } else if (XElemTy->isStructTy()) {
11570 LoadInst *OldVal = Builder.CreateLoad(Ty: XElemTy, Ptr: X.Var, Name: "omp.atomic.read");
11571 const DataLayout &DL = OldVal->getDataLayout();
11572 unsigned LoadSize = DL.getTypeStoreSize(Ty: XElemTy);
11573 OpenMPIRBuilder::AtomicInfo atomicInfo(
11574 &Builder, XElemTy, LoadSize * 8, LoadSize * 8, OldVal->getAlign(),
11575 OldVal->getAlign(), true /* UseLibcall */, AllocaIP, X.Var);
11576 atomicInfo.EmitAtomicStoreLibcall(AO, Source: Expr);
11577 OldVal->eraseFromParent();
11578 } else {
11579 // We need to bitcast and perform atomic op as integers
11580 IntegerType *IntCastTy =
11581 IntegerType::get(C&: M.getContext(), NumBits: XElemTy->getScalarSizeInBits());
11582 Value *ExprCast =
11583 Builder.CreateBitCast(V: Expr, DestTy: IntCastTy, Name: "atomic.src.int.cast");
11584 StoreInst *XSt = Builder.CreateStore(Val: ExprCast, Ptr: X.Var, isVolatile: X.IsVolatile);
11585 XSt->setAtomic(Ordering: AO);
11586 }
11587
11588 checkAndEmitFlushAfterAtomic(Loc, AO, AK: AtomicKind::Write);
11589 return Builder.saveIP();
11590}
11591
11592OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createAtomicUpdate(
11593 const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X,
11594 Value *Expr, AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp,
11595 AtomicUpdateCallbackTy &UpdateOp, bool IsXBinopExpr,
11596 bool IsIgnoreDenormalMode, bool IsFineGrainedMemory, bool IsRemoteMemory) {
11597 assert(!isConflictIP(Loc.IP, AllocaIP) && "IPs must not be ambiguous");
11598 if (!updateToLocation(Loc))
11599 return Loc.IP;
11600
11601 LLVM_DEBUG({
11602 Type *XTy = X.Var->getType();
11603 assert(XTy->isPointerTy() &&
11604 "OMP Atomic expects a pointer to target memory");
11605 Type *XElemTy = X.ElemTy;
11606 assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||
11607 XElemTy->isPointerTy() || XElemTy->isStructTy()) &&
11608 "OMP atomic update expected a scalar or struct type");
11609 });
11610
11611 Expected<std::pair<Value *, Value *>> AtomicResult = emitAtomicUpdate(
11612 AllocaIP, X: X.Var, XElemTy: X.ElemTy, Expr, AO, RMWOp, UpdateOp, VolatileX: X.IsVolatile,
11613 IsXBinopExpr, IsIgnoreDenormalMode, IsFineGrainedMemory, IsRemoteMemory);
11614 if (!AtomicResult)
11615 return AtomicResult.takeError();
11616 checkAndEmitFlushAfterAtomic(Loc, AO, AK: AtomicKind::Update);
11617 return Builder.saveIP();
11618}
11619
11620// FIXME: Duplicating AtomicExpand
11621Value *OpenMPIRBuilder::emitRMWOpAsInstruction(Value *Src1, Value *Src2,
11622 AtomicRMWInst::BinOp RMWOp) {
11623 switch (RMWOp) {
11624 case AtomicRMWInst::Add:
11625 return Builder.CreateAdd(LHS: Src1, RHS: Src2);
11626 case AtomicRMWInst::Sub:
11627 return Builder.CreateSub(LHS: Src1, RHS: Src2);
11628 case AtomicRMWInst::And:
11629 return Builder.CreateAnd(LHS: Src1, RHS: Src2);
11630 case AtomicRMWInst::Nand:
11631 return Builder.CreateNeg(V: Builder.CreateAnd(LHS: Src1, RHS: Src2));
11632 case AtomicRMWInst::Or:
11633 return Builder.CreateOr(LHS: Src1, RHS: Src2);
11634 case AtomicRMWInst::Xor:
11635 return Builder.CreateXor(LHS: Src1, RHS: Src2);
11636 case AtomicRMWInst::Max:
11637 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::smax, LHS: Src1, RHS: Src2);
11638 case AtomicRMWInst::Min:
11639 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::smin, LHS: Src1, RHS: Src2);
11640 case AtomicRMWInst::UMax:
11641 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::umax, LHS: Src1, RHS: Src2);
11642 case AtomicRMWInst::UMin:
11643 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::umin, LHS: Src1, RHS: Src2);
11644 case AtomicRMWInst::FAdd:
11645 return Builder.CreateFAdd(L: Src1, R: Src2);
11646 case AtomicRMWInst::FSub:
11647 return Builder.CreateFSub(L: Src1, R: Src2);
11648 case AtomicRMWInst::FMax:
11649 return Builder.CreateMaxNum(LHS: Src1, RHS: Src2);
11650 case AtomicRMWInst::FMin:
11651 return Builder.CreateMinNum(LHS: Src1, RHS: Src2);
11652 case AtomicRMWInst::FMaximum:
11653 return Builder.CreateMaximum(LHS: Src1, RHS: Src2);
11654 case AtomicRMWInst::FMinimum:
11655 return Builder.CreateMinimum(LHS: Src1, RHS: Src2);
11656 case AtomicRMWInst::Xchg:
11657 case AtomicRMWInst::BAD_BINOP:
11658 case AtomicRMWInst::FMaximumNum:
11659 case AtomicRMWInst::FMinimumNum:
11660 case AtomicRMWInst::UIncWrap:
11661 case AtomicRMWInst::UDecWrap:
11662 case AtomicRMWInst::USubCond:
11663 case AtomicRMWInst::USubSat:
11664 llvm_unreachable("Unsupported atomic update operation");
11665 }
11666 llvm_unreachable("Unsupported atomic update operation");
11667}
11668
11669static AtomicOrdering TransformReleaseAcquireRelease(AtomicOrdering AO) {
11670 // Loads cannot use Release or AcquireRelease ordering. This load is
11671 // just the initial value for the cmpxchg loop; the cmpxchg itself
11672 // retains the original ordering.
11673 AtomicOrdering LoadAO = AO;
11674
11675 if (AO == AtomicOrdering::Release) {
11676 LoadAO = AtomicOrdering::Monotonic;
11677 } else if (AO == AtomicOrdering::AcquireRelease) {
11678 LoadAO = AtomicOrdering::Acquire;
11679 }
11680
11681 return LoadAO;
11682}
11683
11684Expected<std::pair<Value *, Value *>> OpenMPIRBuilder::emitAtomicUpdate(
11685 InsertPointTy AllocaIP, Value *X, Type *XElemTy, Value *Expr,
11686 AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp,
11687 AtomicUpdateCallbackTy &UpdateOp, bool VolatileX, bool IsXBinopExpr,
11688 bool IsIgnoreDenormalMode, bool IsFineGrainedMemory, bool IsRemoteMemory) {
11689 // TODO: handle the case where XElemTy is not byte-sized or not a power of 2.
11690 bool emitRMWOp = false;
11691 switch (RMWOp) {
11692 case AtomicRMWInst::Add:
11693 case AtomicRMWInst::And:
11694 case AtomicRMWInst::Nand:
11695 case AtomicRMWInst::Or:
11696 case AtomicRMWInst::Xor:
11697 case AtomicRMWInst::Xchg:
11698 case AtomicRMWInst::Max:
11699 case AtomicRMWInst::Min:
11700 case AtomicRMWInst::UMax:
11701 case AtomicRMWInst::UMin:
11702 case AtomicRMWInst::FAdd:
11703 case AtomicRMWInst::FMax:
11704 case AtomicRMWInst::FMin:
11705 case AtomicRMWInst::FMaximum:
11706 case AtomicRMWInst::FMinimum:
11707 emitRMWOp = XElemTy;
11708 break;
11709 case AtomicRMWInst::Sub:
11710 case AtomicRMWInst::FSub:
11711 emitRMWOp = (IsXBinopExpr && XElemTy);
11712 break;
11713 default:
11714 emitRMWOp = false;
11715 }
11716 emitRMWOp &=
11717 (XElemTy->isIntegerTy() || XElemTy->isFloatTy() || XElemTy->isDoubleTy());
11718
11719 std::pair<Value *, Value *> Res;
11720 if (emitRMWOp) {
11721 AtomicRMWInst *RMWInst =
11722 Builder.CreateAtomicRMW(Op: RMWOp, Ptr: X, Val: Expr, Align: llvm::MaybeAlign(), Ordering: AO);
11723 if (IsIgnoreDenormalMode)
11724 RMWInst->setMetadata(KindID: llvm::LLVMContext::MD_atomic_ignore_denormal_mode,
11725 Node: llvm::MDNode::get(Context&: Builder.getContext(), MDs: {}));
11726 if (T.isAMDGPU()) {
11727 if (!IsFineGrainedMemory)
11728 RMWInst->setMetadata(Kind: "amdgpu.no.fine.grained.memory",
11729 Node: llvm::MDNode::get(Context&: Builder.getContext(), MDs: {}));
11730 if (!IsRemoteMemory)
11731 RMWInst->setMetadata(Kind: "amdgpu.no.remote.memory",
11732 Node: llvm::MDNode::get(Context&: Builder.getContext(), MDs: {}));
11733 }
11734 Res.first = RMWInst;
11735 // not needed except in case of postfix captures. Generate anyway for
11736 // consistency with the else part. Will be removed with any DCE pass.
11737 // AtomicRMWInst::Xchg does not have a coressponding instruction.
11738 if (RMWOp == AtomicRMWInst::Xchg)
11739 Res.second = Res.first;
11740 else
11741 Res.second = emitRMWOpAsInstruction(Src1: Res.first, Src2: Expr, RMWOp);
11742 } else if (XElemTy->isStructTy()) {
11743 LoadInst *OldVal =
11744 Builder.CreateLoad(Ty: XElemTy, Ptr: X, Name: X->getName() + ".atomic.load");
11745 AtomicOrdering LoadAO = TransformReleaseAcquireRelease(AO);
11746 OldVal->setAtomic(Ordering: LoadAO);
11747 const DataLayout &LoadDL = OldVal->getDataLayout();
11748 unsigned LoadSize = LoadDL.getTypeStoreSize(Ty: XElemTy);
11749
11750 OpenMPIRBuilder::AtomicInfo atomicInfo(
11751 &Builder, XElemTy, LoadSize * 8, LoadSize * 8, OldVal->getAlign(),
11752 OldVal->getAlign(), true /* UseLibcall */, AllocaIP, X);
11753 auto AtomicLoadRes = atomicInfo.EmitAtomicLoadLibcall(AO);
11754 BasicBlock *CurBB = Builder.GetInsertBlock();
11755 Instruction *CurBBTI = CurBB->getTerminatorOrNull();
11756 CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable();
11757 BasicBlock *ExitBB =
11758 CurBB->splitBasicBlock(I: CurBBTI, BBName: X->getName() + ".atomic.exit");
11759 BasicBlock *ContBB = CurBB->splitBasicBlock(I: CurBB->getTerminator(),
11760 BBName: X->getName() + ".atomic.cont");
11761 ContBB->getTerminator()->eraseFromParent();
11762 Builder.restoreIP(IP: AllocaIP);
11763 AllocaInst *NewAtomicAddr = Builder.CreateAlloca(Ty: XElemTy);
11764 NewAtomicAddr->setName(X->getName() + "x.new.val");
11765 Builder.SetInsertPoint(ContBB);
11766 llvm::PHINode *PHI = Builder.CreatePHI(Ty: OldVal->getType(), NumReservedValues: 2);
11767 PHI->addIncoming(V: AtomicLoadRes.first, BB: CurBB);
11768 Value *OldExprVal = PHI;
11769 Expected<Value *> CBResult = UpdateOp(OldExprVal, Builder);
11770 if (!CBResult)
11771 return CBResult.takeError();
11772 Value *Upd = *CBResult;
11773 Builder.CreateStore(Val: Upd, Ptr: NewAtomicAddr);
11774 AtomicOrdering Failure =
11775 llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: AO);
11776 auto Result = atomicInfo.EmitAtomicCompareExchangeLibcall(
11777 ExpectedVal: AtomicLoadRes.second, DesiredVal: NewAtomicAddr, Success: AO, Failure);
11778 LoadInst *PHILoad = Builder.CreateLoad(Ty: XElemTy, Ptr: Result.first);
11779 PHI->addIncoming(V: PHILoad, BB: Builder.GetInsertBlock());
11780 Builder.CreateCondBr(Cond: Result.second, True: ExitBB, False: ContBB);
11781 OldVal->eraseFromParent();
11782 Res.first = OldExprVal;
11783 Res.second = Upd;
11784
11785 if (UnreachableInst *ExitTI =
11786 dyn_cast<UnreachableInst>(Val: ExitBB->getTerminator())) {
11787 CurBBTI->eraseFromParent();
11788 Builder.SetInsertPoint(ExitBB);
11789 } else {
11790 Builder.SetInsertPoint(ExitTI);
11791 }
11792 } else {
11793 IntegerType *IntCastTy =
11794 IntegerType::get(C&: M.getContext(), NumBits: XElemTy->getScalarSizeInBits());
11795 LoadInst *OldVal =
11796 Builder.CreateLoad(Ty: IntCastTy, Ptr: X, Name: X->getName() + ".atomic.load");
11797 AtomicOrdering LoadAO = TransformReleaseAcquireRelease(AO);
11798 OldVal->setAtomic(Ordering: LoadAO);
11799 // CurBB
11800 // | /---\
11801 // ContBB |
11802 // | \---/
11803 // ExitBB
11804 BasicBlock *CurBB = Builder.GetInsertBlock();
11805 Instruction *CurBBTI = CurBB->getTerminatorOrNull();
11806 CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable();
11807 BasicBlock *ExitBB =
11808 CurBB->splitBasicBlock(I: CurBBTI, BBName: X->getName() + ".atomic.exit");
11809 BasicBlock *ContBB = CurBB->splitBasicBlock(I: CurBB->getTerminator(),
11810 BBName: X->getName() + ".atomic.cont");
11811 ContBB->getTerminator()->eraseFromParent();
11812 Builder.restoreIP(IP: AllocaIP);
11813 AllocaInst *NewAtomicAddr = Builder.CreateAlloca(Ty: XElemTy);
11814 NewAtomicAddr->setName(X->getName() + "x.new.val");
11815 Builder.SetInsertPoint(ContBB);
11816 llvm::PHINode *PHI = Builder.CreatePHI(Ty: OldVal->getType(), NumReservedValues: 2);
11817 PHI->addIncoming(V: OldVal, BB: CurBB);
11818 bool IsIntTy = XElemTy->isIntegerTy();
11819 Value *OldExprVal = PHI;
11820 if (!IsIntTy) {
11821 if (XElemTy->isFloatingPointTy()) {
11822 OldExprVal = Builder.CreateBitCast(V: PHI, DestTy: XElemTy,
11823 Name: X->getName() + ".atomic.fltCast");
11824 } else {
11825 OldExprVal = Builder.CreateIntToPtr(V: PHI, DestTy: XElemTy,
11826 Name: X->getName() + ".atomic.ptrCast");
11827 }
11828 }
11829
11830 Expected<Value *> CBResult = UpdateOp(OldExprVal, Builder);
11831 if (!CBResult)
11832 return CBResult.takeError();
11833 Value *Upd = *CBResult;
11834 Builder.CreateStore(Val: Upd, Ptr: NewAtomicAddr);
11835 LoadInst *DesiredVal = Builder.CreateLoad(Ty: IntCastTy, Ptr: NewAtomicAddr);
11836 AtomicOrdering Failure =
11837 llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: AO);
11838 AtomicCmpXchgInst *Result = Builder.CreateAtomicCmpXchg(
11839 Ptr: X, Cmp: PHI, New: DesiredVal, Align: llvm::MaybeAlign(), SuccessOrdering: AO, FailureOrdering: Failure);
11840 Result->setVolatile(VolatileX);
11841 Value *PreviousVal = Builder.CreateExtractValue(Agg: Result, /*Idxs=*/0);
11842 Value *SuccessFailureVal = Builder.CreateExtractValue(Agg: Result, /*Idxs=*/1);
11843 PHI->addIncoming(V: PreviousVal, BB: Builder.GetInsertBlock());
11844 Builder.CreateCondBr(Cond: SuccessFailureVal, True: ExitBB, False: ContBB);
11845
11846 Res.first = OldExprVal;
11847 Res.second = Upd;
11848
11849 // set Insertion point in exit block
11850 if (UnreachableInst *ExitTI =
11851 dyn_cast<UnreachableInst>(Val: ExitBB->getTerminator())) {
11852 CurBBTI->eraseFromParent();
11853 Builder.SetInsertPoint(ExitBB);
11854 } else {
11855 Builder.SetInsertPoint(ExitTI);
11856 }
11857 }
11858
11859 return Res;
11860}
11861
11862OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createAtomicCapture(
11863 const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X,
11864 AtomicOpValue &V, Value *Expr, AtomicOrdering AO,
11865 AtomicRMWInst::BinOp RMWOp, AtomicUpdateCallbackTy &UpdateOp,
11866 bool UpdateExpr, bool IsPostfixUpdate, bool IsXBinopExpr,
11867 bool IsIgnoreDenormalMode, bool IsFineGrainedMemory, bool IsRemoteMemory) {
11868 if (!updateToLocation(Loc))
11869 return Loc.IP;
11870
11871 LLVM_DEBUG({
11872 Type *XTy = X.Var->getType();
11873 assert(XTy->isPointerTy() &&
11874 "OMP Atomic expects a pointer to target memory");
11875 Type *XElemTy = X.ElemTy;
11876 assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||
11877 XElemTy->isPointerTy() || XElemTy->isStructTy()) &&
11878 "OMP atomic capture expected a scalar or struct type");
11879 });
11880
11881 // If UpdateExpr is 'x' updated with some `expr` not based on 'x',
11882 // 'x' is simply atomically rewritten with 'expr'.
11883 AtomicRMWInst::BinOp AtomicOp = (UpdateExpr ? RMWOp : AtomicRMWInst::Xchg);
11884 Expected<std::pair<Value *, Value *>> AtomicResult = emitAtomicUpdate(
11885 AllocaIP, X: X.Var, XElemTy: X.ElemTy, Expr, AO, RMWOp: AtomicOp, UpdateOp, VolatileX: X.IsVolatile,
11886 IsXBinopExpr, IsIgnoreDenormalMode, IsFineGrainedMemory, IsRemoteMemory);
11887 if (!AtomicResult)
11888 return AtomicResult.takeError();
11889 Value *CapturedVal =
11890 (IsPostfixUpdate ? AtomicResult->first : AtomicResult->second);
11891 Builder.CreateStore(Val: CapturedVal, Ptr: V.Var, isVolatile: V.IsVolatile);
11892
11893 checkAndEmitFlushAfterAtomic(Loc, AO, AK: AtomicKind::Capture);
11894 return Builder.saveIP();
11895}
11896
11897OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicCompare(
11898 const LocationDescription &Loc, AtomicOpValue &X, AtomicOpValue &V,
11899 AtomicOpValue &R, Value *E, Value *D, AtomicOrdering AO,
11900 omp::OMPAtomicCompareOp Op, bool IsXBinopExpr, bool IsPostfixUpdate,
11901 bool IsFailOnly, bool IsWeak) {
11902
11903 AtomicOrdering Failure = AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: AO);
11904 return createAtomicCompare(Loc, X, V, R, E, D, AO, Op, IsXBinopExpr,
11905 IsPostfixUpdate, IsFailOnly, Failure, IsWeak);
11906}
11907
11908OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicCompare(
11909 const LocationDescription &Loc, AtomicOpValue &X, AtomicOpValue &V,
11910 AtomicOpValue &R, Value *E, Value *D, AtomicOrdering AO,
11911 omp::OMPAtomicCompareOp Op, bool IsXBinopExpr, bool IsPostfixUpdate,
11912 bool IsFailOnly, AtomicOrdering Failure, bool IsWeak) {
11913
11914 if (!updateToLocation(Loc))
11915 return Loc.IP;
11916
11917 assert(X.Var->getType()->isPointerTy() &&
11918 "OMP atomic expects a pointer to target memory");
11919 // compare capture
11920 if (V.Var) {
11921 assert(V.Var->getType()->isPointerTy() && "v.var must be of pointer type");
11922 assert(V.ElemTy == X.ElemTy && "x and v must be of same type");
11923 }
11924
11925 bool IsInteger = E->getType()->isIntegerTy();
11926
11927 if (Op == OMPAtomicCompareOp::EQ) {
11928 // OldValue and SuccessOrFail are set below and used in the shared V.Var /
11929 // R.Var handling.
11930 Value *OldValue = nullptr;
11931 Value *SuccessOrFail = nullptr;
11932
11933 if (!IsInteger && HandleFPNegZero) {
11934 // IEEE 754 special cases for cmpxchg (which is bitwise):
11935 // 1. -0.0 == +0.0 but they have different bit patterns.
11936 // 2. NaN != NaN but identical NaN bit patterns would match.
11937 //
11938 // CurBB:
11939 // %e_int = bitcast E to intN
11940 // %d_int = bitcast D to intN
11941 // %x_curr = load atomic intN, X
11942 // %x_fp = bitcast %x_curr to FP
11943 // %e_is_nan = fcmp uno E, E
11944 // %x_is_nan = fcmp uno %x_fp, %x_fp
11945 // %either_nan = or %e_is_nan, %x_is_nan
11946 // br %either_nan, NaNBB, NotNaNBB
11947 // NaNBB: ; NaN == anything is always false
11948 // br ExitBB
11949 // NotNaNBB:
11950 // %x_is_zero = fcmp oeq %x_fp, 0.0
11951 // %e_is_zero = fcmp oeq E, 0.0
11952 // %both_zero = and %x_is_zero, %e_is_zero
11953 // br %both_zero, ZeroBB, NormalBB
11954 // ZeroBB: ; both ±0.0 → x = d
11955 // cmpxchg X, %x_curr, %d_int
11956 // br ExitBB
11957 // NormalBB: ; original path
11958 // cmpxchg X, %e_int, %d_int
11959 // br ExitBB
11960 // ExitBB:
11961 // phi merge
11962 IntegerType *IntCastTy =
11963 IntegerType::get(C&: M.getContext(), NumBits: X.ElemTy->getScalarSizeInBits());
11964 Value *EBCast = Builder.CreateBitCast(V: E, DestTy: IntCastTy);
11965 Value *DBCast = Builder.CreateBitCast(V: D, DestTy: IntCastTy);
11966
11967 // Load X atomically.
11968 LoadInst *XCurr = Builder.CreateLoad(Ty: IntCastTy, Ptr: X.Var,
11969 Name: X.Var->getName() + ".atomic.load");
11970 XCurr->setAtomic(Ordering: AtomicOrdering::Monotonic);
11971 Value *XFP = Builder.CreateBitCast(V: XCurr, DestTy: X.ElemTy);
11972
11973 // IEEE 754: NaN != NaN, but cmpxchg would succeed if E and X have
11974 // the same NaN bit pattern. Skip cmpxchg when either is NaN.
11975 Value *EIsNaN = Builder.CreateFCmpUNO(LHS: E, RHS: E, Name: "atomic.e.isnan");
11976 Value *XIsNaN = Builder.CreateFCmpUNO(LHS: XFP, RHS: XFP, Name: "atomic.x.isnan");
11977 Value *EitherNaN = Builder.CreateOr(LHS: EIsNaN, RHS: XIsNaN, Name: "atomic.either.nan");
11978
11979 BasicBlock *CurBB = Builder.GetInsertBlock();
11980 Function *F = CurBB->getParent();
11981 Instruction *CurBBTI = CurBB->getTerminatorOrNull();
11982 CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable();
11983 BasicBlock *ExitBB =
11984 CurBB->splitBasicBlock(I: CurBBTI, BBName: X.Var->getName() + ".atomic.exit");
11985 BasicBlock *NaNBB = BasicBlock::Create(
11986 Context&: M.getContext(), Name: X.Var->getName() + ".atomic.nan", Parent: F, InsertBefore: ExitBB);
11987 BasicBlock *NotNaNBB = BasicBlock::Create(
11988 Context&: M.getContext(), Name: X.Var->getName() + ".atomic.notnan", Parent: F, InsertBefore: ExitBB);
11989 BasicBlock *ZeroBB = BasicBlock::Create(
11990 Context&: M.getContext(), Name: X.Var->getName() + ".atomic.zero", Parent: F, InsertBefore: ExitBB);
11991 BasicBlock *NormalBB = BasicBlock::Create(
11992 Context&: M.getContext(), Name: X.Var->getName() + ".atomic.normal", Parent: F, InsertBefore: ExitBB);
11993
11994 // If either E or X is NaN → NaNBB (always fails), else check for ±0.0.
11995 CurBB->getTerminator()->eraseFromParent();
11996 Builder.SetInsertPoint(CurBB);
11997 Builder.CreateCondBr(Cond: EitherNaN, True: NaNBB, False: NotNaNBB);
11998
11999 // NaNBB: NaN == anything is always false; skip cmpxchg.
12000 Builder.SetInsertPoint(NaNBB);
12001 Builder.CreateBr(Dest: ExitBB);
12002
12003 // NotNaNBB: check both X and E for ±0.0.
12004 Builder.SetInsertPoint(NotNaNBB);
12005 Value *XIsZero =
12006 Builder.CreateFCmpOEQ(LHS: XFP, RHS: ConstantFP::getZero(Ty: X.ElemTy),
12007 Name: X.Var->getName() + ".atomic.xiszero");
12008 Value *EIsZero = Builder.CreateFCmpOEQ(LHS: E, RHS: ConstantFP::getZero(Ty: X.ElemTy),
12009 Name: "atomic.e.iszero");
12010 Value *BothZero = Builder.CreateAnd(LHS: XIsZero, RHS: EIsZero, Name: "atomic.both.zero");
12011 Builder.CreateCondBr(Cond: BothZero, True: ZeroBB, False: NormalBB);
12012
12013 // ZeroBB: cmpxchg with X's loaded bit-pattern.
12014 Builder.SetInsertPoint(ZeroBB);
12015 AtomicCmpXchgInst *ResZero = Builder.CreateAtomicCmpXchg(
12016 Ptr: X.Var, Cmp: XCurr, New: DBCast, Align: MaybeAlign(), SuccessOrdering: AO, FailureOrdering: Failure);
12017 ResZero->setWeak(IsWeak);
12018 Value *OldZero = Builder.CreateExtractValue(Agg: ResZero, /*Idxs=*/0);
12019 Value *OkZero = Builder.CreateExtractValue(Agg: ResZero, /*Idxs=*/1);
12020 Builder.CreateBr(Dest: ExitBB);
12021
12022 // NormalBB: original bitwise cmpxchg.
12023 Builder.SetInsertPoint(NormalBB);
12024 AtomicCmpXchgInst *ResNormal = Builder.CreateAtomicCmpXchg(
12025 Ptr: X.Var, Cmp: EBCast, New: DBCast, Align: MaybeAlign(), SuccessOrdering: AO, FailureOrdering: Failure);
12026 ResNormal->setWeak(IsWeak);
12027 Value *OldNormal = Builder.CreateExtractValue(Agg: ResNormal, /*Idxs=*/0);
12028 Value *OkNormal = Builder.CreateExtractValue(Agg: ResNormal, /*Idxs=*/1);
12029 Builder.CreateBr(Dest: ExitBB);
12030
12031 // ExitBB: merge results from NaN, Zero, and Normal paths.
12032 Builder.SetInsertPoint(ExitBB->begin());
12033 PHINode *OldIntPHI =
12034 Builder.CreatePHI(Ty: IntCastTy, NumReservedValues: 3, Name: X.Var->getName() + ".atomic.old");
12035 OldIntPHI->addIncoming(V: XCurr, BB: NaNBB);
12036 OldIntPHI->addIncoming(V: OldZero, BB: ZeroBB);
12037 OldIntPHI->addIncoming(V: OldNormal, BB: NormalBB);
12038 PHINode *SuccessPHI = Builder.CreatePHI(Ty: Builder.getInt1Ty(), NumReservedValues: 3,
12039 Name: X.Var->getName() + ".atomic.ok");
12040 SuccessPHI->addIncoming(V: Builder.getFalse(), BB: NaNBB);
12041 SuccessPHI->addIncoming(V: OkZero, BB: ZeroBB);
12042 SuccessPHI->addIncoming(V: OkNormal, BB: NormalBB);
12043
12044 if (isa<UnreachableInst>(Val: ExitBB->getTerminator())) {
12045 CurBBTI->eraseFromParent();
12046 Builder.SetInsertPoint(ExitBB);
12047 } else {
12048 Builder.SetInsertPoint(&*ExitBB->getFirstNonPHIIt());
12049 }
12050
12051 OldValue = Builder.CreateBitCast(V: OldIntPHI, DestTy: X.ElemTy,
12052 Name: X.Var->getName() + ".atomic.old.fp");
12053 SuccessOrFail = SuccessPHI;
12054 } else {
12055 AtomicCmpXchgInst *Result = nullptr;
12056 if (!IsInteger) {
12057 IntegerType *IntCastTy =
12058 IntegerType::get(C&: M.getContext(), NumBits: X.ElemTy->getScalarSizeInBits());
12059 Value *EBCast = Builder.CreateBitCast(V: E, DestTy: IntCastTy);
12060 Value *DBCast = Builder.CreateBitCast(V: D, DestTy: IntCastTy);
12061 Result = Builder.CreateAtomicCmpXchg(Ptr: X.Var, Cmp: EBCast, New: DBCast,
12062 Align: MaybeAlign(), SuccessOrdering: AO, FailureOrdering: Failure);
12063 } else {
12064 Result =
12065 Builder.CreateAtomicCmpXchg(Ptr: X.Var, Cmp: E, New: D, Align: MaybeAlign(), SuccessOrdering: AO, FailureOrdering: Failure);
12066 }
12067 Result->setWeak(IsWeak);
12068
12069 if (V.Var) {
12070 OldValue = Builder.CreateExtractValue(Agg: Result, /*Idxs=*/0);
12071 if (!IsInteger)
12072 OldValue = Builder.CreateBitCast(V: OldValue, DestTy: X.ElemTy);
12073 assert(OldValue->getType() == V.ElemTy &&
12074 "OldValue and V must be of same type");
12075 if (IsPostfixUpdate) {
12076 Builder.CreateStore(Val: OldValue, Ptr: V.Var, isVolatile: V.IsVolatile);
12077 } else {
12078 SuccessOrFail = Builder.CreateExtractValue(Agg: Result, /*Idxs=*/1);
12079 if (IsFailOnly) {
12080 BasicBlock *CurBB = Builder.GetInsertBlock();
12081 Instruction *CurBBTI = CurBB->getTerminatorOrNull();
12082 CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable();
12083 BasicBlock *ExitBB = CurBB->splitBasicBlock(
12084 I: CurBBTI, BBName: X.Var->getName() + ".atomic.exit");
12085 BasicBlock *ContBB = CurBB->splitBasicBlock(
12086 I: CurBB->getTerminator(), BBName: X.Var->getName() + ".atomic.cont");
12087 ContBB->getTerminator()->eraseFromParent();
12088 CurBB->getTerminator()->eraseFromParent();
12089
12090 Builder.CreateCondBr(Cond: SuccessOrFail, True: ExitBB, False: ContBB);
12091
12092 Builder.SetInsertPoint(ContBB);
12093 Builder.CreateStore(Val: OldValue, Ptr: V.Var);
12094 Builder.CreateBr(Dest: ExitBB);
12095
12096 if (UnreachableInst *ExitTI =
12097 dyn_cast<UnreachableInst>(Val: ExitBB->getTerminator())) {
12098 CurBBTI->eraseFromParent();
12099 Builder.SetInsertPoint(ExitBB);
12100 } else {
12101 Builder.SetInsertPoint(ExitTI);
12102 }
12103 } else {
12104 Value *CapturedValue =
12105 Builder.CreateSelect(C: SuccessOrFail, True: E, False: OldValue);
12106 Builder.CreateStore(Val: CapturedValue, Ptr: V.Var, isVolatile: V.IsVolatile);
12107 }
12108 }
12109 }
12110 // The comparison result has to be stored.
12111 if (R.Var) {
12112 assert(R.Var->getType()->isPointerTy() &&
12113 "r.var must be of pointer type");
12114 assert(R.ElemTy->isIntegerTy() && "r must be of integral type");
12115
12116 Value *SuccessFailureVal =
12117 Builder.CreateExtractValue(Agg: Result, /*Idxs=*/1);
12118 Value *ResultCast =
12119 R.IsSigned ? Builder.CreateSExt(V: SuccessFailureVal, DestTy: R.ElemTy)
12120 : Builder.CreateZExt(V: SuccessFailureVal, DestTy: R.ElemTy);
12121 Builder.CreateStore(Val: ResultCast, Ptr: R.Var, isVolatile: R.IsVolatile);
12122 }
12123 }
12124
12125 // For the HandleFPNegZero path, handle V.Var and R.Var using the
12126 // pre-computed OldValue and SuccessOrFail.
12127 if (HandleFPNegZero && !IsInteger) {
12128 if (V.Var) {
12129 assert(OldValue->getType() == V.ElemTy &&
12130 "OldValue and V must be of same type");
12131 if (IsPostfixUpdate) {
12132 Builder.CreateStore(Val: OldValue, Ptr: V.Var, isVolatile: V.IsVolatile);
12133 } else {
12134 if (IsFailOnly) {
12135 BasicBlock *CurBB = Builder.GetInsertBlock();
12136 Instruction *CurBBTI = CurBB->getTerminatorOrNull();
12137 CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable();
12138 BasicBlock *ExitBB = CurBB->splitBasicBlock(
12139 I: CurBBTI, BBName: X.Var->getName() + ".atomic.exit");
12140 BasicBlock *ContBB = CurBB->splitBasicBlock(
12141 I: CurBB->getTerminator(), BBName: X.Var->getName() + ".atomic.cont");
12142 ContBB->getTerminator()->eraseFromParent();
12143 CurBB->getTerminator()->eraseFromParent();
12144
12145 Builder.CreateCondBr(Cond: SuccessOrFail, True: ExitBB, False: ContBB);
12146
12147 Builder.SetInsertPoint(ContBB);
12148 Builder.CreateStore(Val: OldValue, Ptr: V.Var);
12149 Builder.CreateBr(Dest: ExitBB);
12150
12151 if (UnreachableInst *ExitTI =
12152 dyn_cast<UnreachableInst>(Val: ExitBB->getTerminator())) {
12153 CurBBTI->eraseFromParent();
12154 Builder.SetInsertPoint(ExitBB);
12155 } else {
12156 Builder.SetInsertPoint(ExitTI);
12157 }
12158 } else {
12159 Value *CapturedValue =
12160 Builder.CreateSelect(C: SuccessOrFail, True: E, False: OldValue);
12161 Builder.CreateStore(Val: CapturedValue, Ptr: V.Var, isVolatile: V.IsVolatile);
12162 }
12163 }
12164 }
12165 // The comparison result has to be stored.
12166 if (R.Var) {
12167 assert(R.Var->getType()->isPointerTy() &&
12168 "r.var must be of pointer type");
12169 assert(R.ElemTy->isIntegerTy() && "r must be of integral type");
12170
12171 Value *ResultCast = R.IsSigned
12172 ? Builder.CreateSExt(V: SuccessOrFail, DestTy: R.ElemTy)
12173 : Builder.CreateZExt(V: SuccessOrFail, DestTy: R.ElemTy);
12174 Builder.CreateStore(Val: ResultCast, Ptr: R.Var, isVolatile: R.IsVolatile);
12175 }
12176 }
12177 } else {
12178 assert((Op == OMPAtomicCompareOp::MAX || Op == OMPAtomicCompareOp::MIN) &&
12179 "Op should be either max or min at this point");
12180 assert(!IsFailOnly && "IsFailOnly is only valid when the comparison is ==");
12181
12182 // Reverse the ordop as the OpenMP forms are different from LLVM forms.
12183 // Let's take max as example.
12184 // OpenMP form:
12185 // x = x > expr ? expr : x;
12186 // LLVM form:
12187 // *ptr = *ptr > val ? *ptr : val;
12188 // We need to transform to LLVM form.
12189 // x = x <= expr ? x : expr;
12190 AtomicRMWInst::BinOp NewOp;
12191 if (IsXBinopExpr) {
12192 if (IsInteger) {
12193 if (X.IsSigned)
12194 NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::Min
12195 : AtomicRMWInst::Max;
12196 else
12197 NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::UMin
12198 : AtomicRMWInst::UMax;
12199 } else {
12200 NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::FMin
12201 : AtomicRMWInst::FMax;
12202 }
12203 } else {
12204 if (IsInteger) {
12205 if (X.IsSigned)
12206 NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::Max
12207 : AtomicRMWInst::Min;
12208 else
12209 NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::UMax
12210 : AtomicRMWInst::UMin;
12211 } else {
12212 NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::FMax
12213 : AtomicRMWInst::FMin;
12214 }
12215 }
12216
12217 AtomicRMWInst *OldValue =
12218 Builder.CreateAtomicRMW(Op: NewOp, Ptr: X.Var, Val: E, Align: MaybeAlign(), Ordering: AO);
12219 if (V.Var) {
12220 Value *CapturedValue = nullptr;
12221 if (IsPostfixUpdate) {
12222 CapturedValue = OldValue;
12223 } else {
12224 CmpInst::Predicate Pred;
12225 switch (NewOp) {
12226 case AtomicRMWInst::Max:
12227 Pred = CmpInst::ICMP_SGT;
12228 break;
12229 case AtomicRMWInst::UMax:
12230 Pred = CmpInst::ICMP_UGT;
12231 break;
12232 case AtomicRMWInst::FMax:
12233 Pred = CmpInst::FCMP_OGT;
12234 break;
12235 case AtomicRMWInst::Min:
12236 Pred = CmpInst::ICMP_SLT;
12237 break;
12238 case AtomicRMWInst::UMin:
12239 Pred = CmpInst::ICMP_ULT;
12240 break;
12241 case AtomicRMWInst::FMin:
12242 Pred = CmpInst::FCMP_OLT;
12243 break;
12244 default:
12245 llvm_unreachable("unexpected comparison op");
12246 }
12247 Value *NonAtomicCmp = Builder.CreateCmp(Pred, LHS: OldValue, RHS: E);
12248 CapturedValue = Builder.CreateSelect(C: NonAtomicCmp, True: E, False: OldValue);
12249 }
12250 Builder.CreateStore(Val: CapturedValue, Ptr: V.Var, isVolatile: V.IsVolatile);
12251 }
12252 }
12253
12254 checkAndEmitFlushAfterAtomic(Loc, AO, AK: AtomicKind::Compare);
12255
12256 return Builder.saveIP();
12257}
12258
12259OpenMPIRBuilder::InsertPointOrErrorTy
12260OpenMPIRBuilder::createTeams(const LocationDescription &Loc,
12261 BodyGenCallbackTy BodyGenCB, Value *NumTeamsLower,
12262 Value *NumTeamsUpper, Value *ThreadLimit,
12263 Value *IfExpr) {
12264 if (!updateToLocation(Loc))
12265 return InsertPointTy();
12266
12267 uint32_t SrcLocStrSize;
12268 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
12269 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
12270 Function *CurrentFunction = Builder.GetInsertBlock()->getParent();
12271
12272 // Outer allocation basicblock is the entry block of the current function.
12273 BasicBlock &OuterAllocaBB = CurrentFunction->getEntryBlock();
12274 if (&OuterAllocaBB == Builder.GetInsertBlock()) {
12275 BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, Name: "teams.entry");
12276 Builder.SetInsertPoint(BodyBB->begin());
12277 }
12278
12279 // The current basic block is split into four basic blocks. After outlining,
12280 // they will be mapped as follows:
12281 // ```
12282 // def current_fn() {
12283 // current_basic_block:
12284 // br label %teams.exit
12285 // teams.exit:
12286 // ; instructions after teams
12287 // }
12288 //
12289 // def outlined_fn() {
12290 // teams.alloca:
12291 // br label %teams.body
12292 // teams.body:
12293 // ; instructions within teams body
12294 // }
12295 // ```
12296 BasicBlock *ExitBB = splitBB(Builder, /*CreateBranch=*/true, Name: "teams.exit");
12297 BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, Name: "teams.body");
12298 BasicBlock *AllocaBB =
12299 splitBB(Builder, /*CreateBranch=*/true, Name: "teams.alloca");
12300
12301 bool SubClausesPresent =
12302 (NumTeamsLower || NumTeamsUpper || ThreadLimit || IfExpr);
12303 // Push num_teams
12304 if (!Config.isTargetDevice() && SubClausesPresent) {
12305 assert((NumTeamsLower == nullptr || NumTeamsUpper != nullptr) &&
12306 "if lowerbound is non-null, then upperbound must also be non-null "
12307 "for bounds on num_teams");
12308
12309 if (NumTeamsUpper == nullptr)
12310 NumTeamsUpper = Builder.getInt32(C: 0);
12311
12312 if (NumTeamsLower == nullptr)
12313 NumTeamsLower = NumTeamsUpper;
12314
12315 if (IfExpr) {
12316 assert(IfExpr->getType()->isIntegerTy() &&
12317 "argument to if clause must be an integer value");
12318
12319 // upper = ifexpr ? upper : 1
12320 if (IfExpr->getType() != Int1)
12321 IfExpr = Builder.CreateICmpNE(LHS: IfExpr,
12322 RHS: ConstantInt::get(Ty: IfExpr->getType(), V: 0));
12323 NumTeamsUpper = Builder.CreateSelect(
12324 C: IfExpr, True: NumTeamsUpper, False: Builder.getInt32(C: 1), Name: "numTeamsUpper");
12325
12326 // lower = ifexpr ? lower : 1
12327 NumTeamsLower = Builder.CreateSelect(
12328 C: IfExpr, True: NumTeamsLower, False: Builder.getInt32(C: 1), Name: "numTeamsLower");
12329 }
12330
12331 if (ThreadLimit == nullptr)
12332 ThreadLimit = Builder.getInt32(C: 0);
12333
12334 // The __kmpc_push_num_teams_51 function expects int32 as the arguments. So,
12335 // truncate or sign extend the passed values to match the int32 parameters.
12336 Value *NumTeamsLowerInt32 =
12337 Builder.CreateSExtOrTrunc(V: NumTeamsLower, DestTy: Builder.getInt32Ty());
12338 Value *NumTeamsUpperInt32 =
12339 Builder.CreateSExtOrTrunc(V: NumTeamsUpper, DestTy: Builder.getInt32Ty());
12340 Value *ThreadLimitInt32 =
12341 Builder.CreateSExtOrTrunc(V: ThreadLimit, DestTy: Builder.getInt32Ty());
12342
12343 Value *ThreadNum = getOrCreateThreadID(Ident);
12344
12345 createRuntimeFunctionCall(
12346 Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_push_num_teams_51),
12347 Args: {Ident, ThreadNum, NumTeamsLowerInt32, NumTeamsUpperInt32,
12348 ThreadLimitInt32});
12349 }
12350 // Generate the body of teams.
12351 InsertPointTy AllocaIP(AllocaBB->begin());
12352 InsertPointTy CodeGenIP(BodyBB->begin());
12353 if (Error Err = BodyGenCB(AllocaIP, CodeGenIP, ExitBB))
12354 return Err;
12355
12356 auto OI = std::make_unique<OutlineInfo>();
12357 OI->EntryBB = AllocaBB;
12358 OI->ExitBB = ExitBB;
12359 OI->OuterAllocBB = &OuterAllocaBB;
12360
12361 // Insert fake values for global tid and bound tid.
12362 SmallVector<Instruction *, 8> ToBeDeleted;
12363 InsertPointTy OuterAllocaIP(OuterAllocaBB.begin());
12364 OI->ExcludeArgsFromAggregate.push_back(Elt: createFakeIntVal(
12365 Builder, OuterAllocaIP, ToBeDeleted, InnerAllocaIP: AllocaIP, Name: "gid", AsPtr: true));
12366 OI->ExcludeArgsFromAggregate.push_back(Elt: createFakeIntVal(
12367 Builder, OuterAllocaIP, ToBeDeleted, InnerAllocaIP: AllocaIP, Name: "tid", AsPtr: true));
12368
12369 auto HostPostOutlineCB = [this, Ident,
12370 ToBeDeleted](Function &OutlinedFn) mutable {
12371 // The stale call instruction will be replaced with a new call instruction
12372 // for runtime call with the outlined function.
12373
12374 assert(OutlinedFn.hasOneUse() &&
12375 "there must be a single user for the outlined function");
12376 CallInst *StaleCI = cast<CallInst>(Val: OutlinedFn.user_back());
12377 ToBeDeleted.push_back(Elt: StaleCI);
12378
12379 assert((OutlinedFn.arg_size() == 2 || OutlinedFn.arg_size() == 3) &&
12380 "Outlined function must have two or three arguments only");
12381
12382 bool HasShared = OutlinedFn.arg_size() == 3;
12383
12384 OutlinedFn.getArg(i: 0)->setName("global.tid.ptr");
12385 OutlinedFn.getArg(i: 1)->setName("bound.tid.ptr");
12386 if (HasShared)
12387 OutlinedFn.getArg(i: 2)->setName("data");
12388
12389 // Call to the runtime function for teams in the current function.
12390 assert(StaleCI && "Error while outlining - no CallInst user found for the "
12391 "outlined function.");
12392 Builder.SetInsertPoint(StaleCI);
12393 SmallVector<Value *> Args = {
12394 Ident, Builder.getInt32(C: StaleCI->arg_size() - 2), &OutlinedFn};
12395 if (HasShared)
12396 Args.push_back(Elt: StaleCI->getArgOperand(i: 2));
12397 createRuntimeFunctionCall(
12398 Callee: getOrCreateRuntimeFunctionPtr(
12399 FnID: omp::RuntimeFunction::OMPRTL___kmpc_fork_teams),
12400 Args);
12401
12402 Builder.ClearInsertionPoint();
12403 for (Instruction *I : llvm::reverse(C&: ToBeDeleted))
12404 I->eraseFromParent();
12405 };
12406
12407 if (!Config.isTargetDevice())
12408 OI->PostOutlineCB = HostPostOutlineCB;
12409
12410 addOutlineInfo(OI: std::move(OI));
12411
12412 Builder.SetInsertPoint(ExitBB);
12413
12414 return Builder.saveIP();
12415}
12416
12417OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createDistribute(
12418 const LocationDescription &Loc, InsertPointTy OuterAllocIP,
12419 ArrayRef<BasicBlock *> OuterDeallocBlocks, BodyGenCallbackTy BodyGenCB) {
12420 if (!updateToLocation(Loc))
12421 return InsertPointTy();
12422
12423 BasicBlock *OuterAllocaBB = OuterAllocIP.getNodeParent();
12424
12425 if (OuterAllocaBB == Builder.GetInsertBlock()) {
12426 BasicBlock *BodyBB =
12427 splitBB(Builder, /*CreateBranch=*/true, Name: "distribute.entry");
12428 Builder.SetInsertPoint(BodyBB->begin());
12429 }
12430 BasicBlock *ExitBB =
12431 splitBB(Builder, /*CreateBranch=*/true, Name: "distribute.exit");
12432 BasicBlock *BodyBB =
12433 splitBB(Builder, /*CreateBranch=*/true, Name: "distribute.body");
12434 BasicBlock *AllocaBB =
12435 splitBB(Builder, /*CreateBranch=*/true, Name: "distribute.alloca");
12436
12437 // Generate the body of distribute clause
12438 InsertPointTy AllocaIP(AllocaBB->begin());
12439 InsertPointTy CodeGenIP(BodyBB->begin());
12440 if (Error Err = BodyGenCB(AllocaIP, CodeGenIP, ExitBB))
12441 return Err;
12442
12443 // When using target we use different runtime functions which require a
12444 // callback.
12445 if (Config.isTargetDevice()) {
12446 auto OI = std::make_unique<OutlineInfo>();
12447 OI->OuterAllocBB = OuterAllocIP.getNodeParent();
12448 OI->EntryBB = AllocaBB;
12449 OI->ExitBB = ExitBB;
12450 OI->OuterDeallocBBs.reserve(N: OuterDeallocBlocks.size());
12451 copy(Range&: OuterDeallocBlocks, Out: OI->OuterDeallocBBs.end());
12452
12453 addOutlineInfo(OI: std::move(OI));
12454 }
12455 Builder.SetInsertPoint(ExitBB);
12456
12457 return Builder.saveIP();
12458}
12459
12460GlobalVariable *
12461OpenMPIRBuilder::createOffloadMapnames(SmallVectorImpl<llvm::Constant *> &Names,
12462 std::string VarName) {
12463 llvm::Constant *MapNamesArrayInit = llvm::ConstantArray::get(
12464 T: llvm::ArrayType::get(ElementType: llvm::PointerType::getUnqual(C&: M.getContext()),
12465 NumElements: Names.size()),
12466 V: Names);
12467 auto *MapNamesArrayGlobal = new llvm::GlobalVariable(
12468 M, MapNamesArrayInit->getType(),
12469 /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage, MapNamesArrayInit,
12470 VarName);
12471 return MapNamesArrayGlobal;
12472}
12473
12474// Create all simple and struct types exposed by the runtime and remember
12475// the llvm::PointerTypes of them for easy access later.
12476void OpenMPIRBuilder::initializeTypes(Module &M) {
12477 LLVMContext &Ctx = M.getContext();
12478 StructType *T;
12479 unsigned DefaultTargetAS = Config.getDefaultTargetAS();
12480 unsigned ProgramAS = M.getDataLayout().getProgramAddressSpace();
12481#define OMP_TYPE(VarName, InitValue) VarName = InitValue;
12482#define OMP_ARRAY_TYPE(VarName, ElemTy, ArraySize) \
12483 VarName##Ty = ArrayType::get(ElemTy, ArraySize); \
12484 VarName##PtrTy = PointerType::get(Ctx, DefaultTargetAS);
12485#define OMP_FUNCTION_TYPE(VarName, IsVarArg, ReturnType, ...) \
12486 VarName = FunctionType::get(ReturnType, {__VA_ARGS__}, IsVarArg); \
12487 VarName##Ptr = PointerType::get(Ctx, ProgramAS);
12488#define OMP_STRUCT_TYPE(VarName, StructName, Packed, ...) \
12489 T = StructType::getTypeByName(Ctx, StructName); \
12490 if (!T) \
12491 T = StructType::create(Ctx, {__VA_ARGS__}, StructName, Packed); \
12492 VarName = T; \
12493 VarName##Ptr = PointerType::get(Ctx, DefaultTargetAS);
12494#include "llvm/Frontend/OpenMP/OMPKinds.def"
12495}
12496
12497void OpenMPIRBuilder::OutlineInfo::collectBlocks(
12498 SmallPtrSetImpl<BasicBlock *> &BlockSet,
12499 SmallVectorImpl<BasicBlock *> &BlockVector) {
12500 SmallVector<BasicBlock *, 32> Worklist;
12501 BlockSet.insert(Ptr: EntryBB);
12502 BlockSet.insert(Ptr: ExitBB);
12503
12504 Worklist.push_back(Elt: EntryBB);
12505 while (!Worklist.empty()) {
12506 BasicBlock *BB = Worklist.pop_back_val();
12507 BlockVector.push_back(Elt: BB);
12508 for (BasicBlock *SuccBB : successors(BB))
12509 if (BlockSet.insert(Ptr: SuccBB).second)
12510 Worklist.push_back(Elt: SuccBB);
12511 }
12512}
12513
12514std::unique_ptr<CodeExtractor>
12515OpenMPIRBuilder::OutlineInfo::createCodeExtractor(ArrayRef<BasicBlock *> Blocks,
12516 bool ArgsInZeroAddressSpace,
12517 Twine Suffix) {
12518 return std::make_unique<CodeExtractor>(
12519 args&: Blocks, /* DominatorTree */ args: nullptr,
12520 /* AggregateArgs */ args: true,
12521 /* BlockFrequencyInfo */ args: nullptr,
12522 /* BranchProbabilityInfo */ args: nullptr,
12523 /* AssumptionCache */ args: nullptr,
12524 /* AllowVarArgs */ args: true,
12525 /* AllowAlloca */ args: true,
12526 /* AllocationBlock*/ args&: OuterAllocBB,
12527 /* DeallocationBlocks */ args: ArrayRef<BasicBlock *>(),
12528 /* Suffix */ args: Suffix.str(), args&: ArgsInZeroAddressSpace);
12529}
12530
12531std::unique_ptr<CodeExtractor> DeviceSharedMemOutlineInfo::createCodeExtractor(
12532 ArrayRef<BasicBlock *> Blocks, bool ArgsInZeroAddressSpace, Twine Suffix) {
12533 return std::make_unique<DeviceSharedMemCodeExtractor>(
12534 args&: OMPBuilder, args&: Blocks, /* DominatorTree */ args: nullptr,
12535 /* AggregateArgs */ args: true,
12536 /* BlockFrequencyInfo */ args: nullptr,
12537 /* BranchProbabilityInfo */ args: nullptr,
12538 /* AssumptionCache */ args: nullptr,
12539 /* AllowVarArgs */ args: true,
12540 /* AllowAlloca */ args: true,
12541 /* AllocationBlock*/ args&: OuterAllocBB,
12542 /* DeallocationBlocks */ args: OuterDeallocBBs.empty()
12543 ? SmallVector<BasicBlock *>{ExitBB}
12544 : OuterDeallocBBs,
12545 /* Suffix */ args: Suffix.str(), args&: ArgsInZeroAddressSpace);
12546}
12547
12548void OpenMPIRBuilder::createOffloadEntry(Constant *ID, Constant *Addr,
12549 uint64_t Size, int32_t Flags,
12550 GlobalValue::LinkageTypes,
12551 StringRef Name) {
12552 if (!Config.isGPU()) {
12553 llvm::offloading::emitOffloadingEntry(
12554 M, Kind: object::OffloadKind::OFK_OpenMP, Addr: ID,
12555 Name: Name.empty() ? Addr->getName() : Name, Size, Flags, /*Data=*/0);
12556 return;
12557 }
12558 // TODO: Add support for global variables on the device after declare target
12559 // support.
12560 Function *Fn = dyn_cast<Function>(Val: Addr);
12561 if (!Fn)
12562 return;
12563
12564 // Add a function attribute for the kernel.
12565 Fn->addFnAttr(Kind: "kernel");
12566 if (T.isAMDGCN())
12567 Fn->addFnAttr(Kind: "uniform-work-group-size");
12568 Fn->addFnAttr(Kind: Attribute::MustProgress);
12569}
12570
12571// We only generate metadata for function that contain target regions.
12572void OpenMPIRBuilder::createOffloadEntriesAndInfoMetadata(
12573 EmitMetadataErrorReportFunctionTy &ErrorFn) {
12574
12575 // If there are no entries, we don't need to do anything.
12576 if (OffloadInfoManager.empty())
12577 return;
12578
12579 LLVMContext &C = M.getContext();
12580 SmallVector<std::pair<const OffloadEntriesInfoManager::OffloadEntryInfo *,
12581 TargetRegionEntryInfo>,
12582 16>
12583 OrderedEntries(OffloadInfoManager.size());
12584
12585 // Auxiliary methods to create metadata values and strings.
12586 auto &&GetMDInt = [this](unsigned V) {
12587 return ConstantAsMetadata::get(C: ConstantInt::get(Ty: Builder.getInt32Ty(), V));
12588 };
12589
12590 auto &&GetMDString = [&C](StringRef V) { return MDString::get(Context&: C, Str: V); };
12591
12592 // Create the offloading info metadata node.
12593 NamedMDNode *MD = M.getOrInsertNamedMetadata(Name: "omp_offload.info");
12594 auto &&TargetRegionMetadataEmitter =
12595 [&C, MD, &OrderedEntries, &GetMDInt, &GetMDString](
12596 const TargetRegionEntryInfo &EntryInfo,
12597 const OffloadEntriesInfoManager::OffloadEntryInfoTargetRegion &E) {
12598 // Generate metadata for target regions. Each entry of this metadata
12599 // contains:
12600 // - Entry 0 -> Kind of this type of metadata (0).
12601 // - Entry 1 -> Device ID of the file where the entry was identified.
12602 // - Entry 2 -> File ID of the file where the entry was identified.
12603 // - Entry 3 -> Mangled name of the function where the entry was
12604 // identified.
12605 // - Entry 4 -> Line in the file where the entry was identified.
12606 // - Entry 5 -> Count of regions at this DeviceID/FilesID/Line.
12607 // - Entry 6 -> Order the entry was created.
12608 // The first element of the metadata node is the kind.
12609 Metadata *Ops[] = {
12610 GetMDInt(E.getKind()), GetMDInt(EntryInfo.DeviceID),
12611 GetMDInt(EntryInfo.FileID), GetMDString(EntryInfo.ParentName),
12612 GetMDInt(EntryInfo.Line), GetMDInt(EntryInfo.Count),
12613 GetMDInt(E.getOrder())};
12614
12615 // Save this entry in the right position of the ordered entries array.
12616 OrderedEntries[E.getOrder()] = std::make_pair(x: &E, y: EntryInfo);
12617
12618 // Add metadata to the named metadata node.
12619 MD->addOperand(M: MDNode::get(Context&: C, MDs: Ops));
12620 };
12621
12622 OffloadInfoManager.actOnTargetRegionEntriesInfo(Action: TargetRegionMetadataEmitter);
12623
12624 // Create function that emits metadata for each device global variable entry;
12625 auto &&DeviceGlobalVarMetadataEmitter =
12626 [&C, &OrderedEntries, &GetMDInt, &GetMDString, MD](
12627 StringRef MangledName,
12628 const OffloadEntriesInfoManager::OffloadEntryInfoDeviceGlobalVar &E) {
12629 // Generate metadata for global variables. Each entry of this metadata
12630 // contains:
12631 // - Entry 0 -> Kind of this type of metadata (1).
12632 // - Entry 1 -> Mangled name of the variable.
12633 // - Entry 2 -> Declare target kind.
12634 // - Entry 3 -> Order the entry was created.
12635 // The first element of the metadata node is the kind.
12636 Metadata *Ops[] = {GetMDInt(E.getKind()), GetMDString(MangledName),
12637 GetMDInt(E.getFlags()), GetMDInt(E.getOrder())};
12638
12639 // Save this entry in the right position of the ordered entries array.
12640 TargetRegionEntryInfo varInfo(MangledName, 0, 0, 0);
12641 OrderedEntries[E.getOrder()] = std::make_pair(x: &E, y&: varInfo);
12642
12643 // Add metadata to the named metadata node.
12644 MD->addOperand(M: MDNode::get(Context&: C, MDs: Ops));
12645 };
12646
12647 OffloadInfoManager.actOnDeviceGlobalVarEntriesInfo(
12648 Action: DeviceGlobalVarMetadataEmitter);
12649
12650 for (const auto &E : OrderedEntries) {
12651 assert(E.first && "All ordered entries must exist!");
12652 if (const auto *CE =
12653 dyn_cast<OffloadEntriesInfoManager::OffloadEntryInfoTargetRegion>(
12654 Val: E.first)) {
12655 if (!CE->getID() || !CE->getAddress()) {
12656 // Do not blame the entry if the parent funtion is not emitted.
12657 TargetRegionEntryInfo EntryInfo = E.second;
12658 StringRef FnName = EntryInfo.ParentName;
12659 if (!M.getNamedValue(Name: FnName))
12660 continue;
12661 ErrorFn(EMIT_MD_TARGET_REGION_ERROR, EntryInfo);
12662 continue;
12663 }
12664 createOffloadEntry(ID: CE->getID(), Addr: CE->getAddress(),
12665 /*Size=*/0, Flags: CE->getFlags(),
12666 GlobalValue::WeakAnyLinkage);
12667 } else if (const auto *CE = dyn_cast<
12668 OffloadEntriesInfoManager::OffloadEntryInfoDeviceGlobalVar>(
12669 Val: E.first)) {
12670 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind Flags =
12671 static_cast<OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind>(
12672 CE->getFlags());
12673 switch (Flags) {
12674 case OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter:
12675 case OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo:
12676 if (Config.isTargetDevice() && Config.hasRequiresUnifiedSharedMemory())
12677 continue;
12678 if (!CE->getAddress()) {
12679 ErrorFn(EMIT_MD_DECLARE_TARGET_ERROR, E.second);
12680 continue;
12681 }
12682 // The vaiable has no definition - no need to add the entry.
12683 if (CE->getVarSize() == 0)
12684 continue;
12685 break;
12686 case OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink:
12687 assert(((Config.isTargetDevice() && !CE->getAddress()) ||
12688 (!Config.isTargetDevice() && CE->getAddress())) &&
12689 "Declaret target link address is set.");
12690 if (Config.isTargetDevice())
12691 continue;
12692 if (!CE->getAddress()) {
12693 ErrorFn(EMIT_MD_GLOBAL_VAR_LINK_ERROR, TargetRegionEntryInfo());
12694 continue;
12695 }
12696 break;
12697 case OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect:
12698 case OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirectVTable:
12699 if (!CE->getAddress()) {
12700 ErrorFn(EMIT_MD_GLOBAL_VAR_INDIRECT_ERROR, E.second);
12701 continue;
12702 }
12703 break;
12704 default:
12705 break;
12706 }
12707
12708 // Hidden or internal symbols on the device are not externally visible.
12709 // We should not attempt to register them by creating an offloading
12710 // entry. Indirect variables are handled separately on the device.
12711 if (auto *GV = dyn_cast<GlobalValue>(Val: CE->getAddress()))
12712 if ((GV->hasLocalLinkage() || GV->hasHiddenVisibility()) &&
12713 (Flags !=
12714 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect &&
12715 Flags != OffloadEntriesInfoManager::
12716 OMPTargetGlobalVarEntryIndirectVTable))
12717 continue;
12718
12719 // Indirect globals need to use a special name that doesn't match the name
12720 // of the associated host global.
12721 if (Flags == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect ||
12722 Flags ==
12723 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirectVTable)
12724 createOffloadEntry(ID: CE->getAddress(), Addr: CE->getAddress(), Size: CE->getVarSize(),
12725 Flags, CE->getLinkage(), Name: CE->getVarName());
12726 else
12727 createOffloadEntry(ID: CE->getAddress(), Addr: CE->getAddress(), Size: CE->getVarSize(),
12728 Flags, CE->getLinkage());
12729
12730 } else {
12731 llvm_unreachable("Unsupported entry kind.");
12732 }
12733 }
12734
12735 // Emit requires directive globals to a special entry so the runtime can
12736 // register them when the device image is loaded.
12737 // TODO: This reduces the offloading entries to a 32-bit integer. Offloading
12738 // entries should be redesigned to better suit this use-case.
12739 if (Config.hasRequiresFlags() && !Config.isTargetDevice())
12740 offloading::emitOffloadingEntry(
12741 M, Kind: object::OffloadKind::OFK_OpenMP,
12742 Addr: Constant::getNullValue(Ty: PointerType::getUnqual(C&: M.getContext())),
12743 Name: ".requires", /*Size=*/0,
12744 Flags: OffloadEntriesInfoManager::OMPTargetGlobalRegisterRequires,
12745 Data: Config.getRequiresFlags());
12746}
12747
12748void TargetRegionEntryInfo::getTargetRegionEntryFnName(
12749 SmallVectorImpl<char> &Name, StringRef ParentName, unsigned DeviceID,
12750 unsigned FileID, unsigned Line, unsigned Count) {
12751 raw_svector_ostream OS(Name);
12752 OS << KernelNamePrefix << llvm::format(Fmt: "%x", Vals: DeviceID)
12753 << llvm::format(Fmt: "_%x_", Vals: FileID) << ParentName << "_l" << Line;
12754 if (Count)
12755 OS << "_" << Count;
12756}
12757
12758void OffloadEntriesInfoManager::getTargetRegionEntryFnName(
12759 SmallVectorImpl<char> &Name, const TargetRegionEntryInfo &EntryInfo) {
12760 unsigned NewCount = getTargetRegionEntryInfoCount(EntryInfo);
12761 TargetRegionEntryInfo::getTargetRegionEntryFnName(
12762 Name, ParentName: EntryInfo.ParentName, DeviceID: EntryInfo.DeviceID, FileID: EntryInfo.FileID,
12763 Line: EntryInfo.Line, Count: NewCount);
12764}
12765
12766TargetRegionEntryInfo
12767OpenMPIRBuilder::getTargetEntryUniqueInfo(FileIdentifierInfoCallbackTy CallBack,
12768 vfs::FileSystem &VFS,
12769 StringRef ParentName) {
12770 sys::fs::UniqueID ID(0xdeadf17e, 0);
12771 auto FileIDInfo = CallBack();
12772 uint64_t FileID = 0;
12773 if (ErrorOr<vfs::Status> Status = VFS.status(Path: std::get<0>(t&: FileIDInfo))) {
12774 ID = Status->getUniqueID();
12775 FileID = Status->getUniqueID().getFile();
12776 } else {
12777 // If the inode ID could not be determined, create a hash value
12778 // the current file name and use that as an ID.
12779 FileID = hash_value(arg: std::get<0>(t&: FileIDInfo));
12780 }
12781
12782 return TargetRegionEntryInfo(ParentName, ID.getDevice(), FileID,
12783 std::get<1>(t&: FileIDInfo));
12784}
12785
12786unsigned OpenMPIRBuilder::getFlagMemberOffset() {
12787 unsigned Offset = 0;
12788 for (uint64_t Remain =
12789 static_cast<std::underlying_type_t<omp::OpenMPOffloadMappingFlags>>(
12790 omp::OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);
12791 !(Remain & 1); Remain = Remain >> 1)
12792 Offset++;
12793 return Offset;
12794}
12795
12796omp::OpenMPOffloadMappingFlags
12797OpenMPIRBuilder::getMemberOfFlag(unsigned Position) {
12798 // Rotate by getFlagMemberOffset() bits.
12799 return static_cast<omp::OpenMPOffloadMappingFlags>(((uint64_t)Position + 1)
12800 << getFlagMemberOffset());
12801}
12802
12803void OpenMPIRBuilder::setCorrectMemberOfFlag(
12804 omp::OpenMPOffloadMappingFlags &Flags,
12805 omp::OpenMPOffloadMappingFlags MemberOfFlag) {
12806 // If the entry is PTR_AND_OBJ but has not been marked with the special
12807 // placeholder value 0xFFFF in the MEMBER_OF field, then it should not be
12808 // marked as MEMBER_OF.
12809 if (static_cast<std::underlying_type_t<omp::OpenMPOffloadMappingFlags>>(
12810 Flags & omp::OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ) &&
12811 static_cast<std::underlying_type_t<omp::OpenMPOffloadMappingFlags>>(
12812 (Flags & omp::OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) !=
12813 omp::OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF))
12814 return;
12815
12816 // Entries with ATTACH are not members-of anything. They are handled
12817 // separately by the runtime after other maps have been handled.
12818 if (static_cast<std::underlying_type_t<omp::OpenMPOffloadMappingFlags>>(
12819 Flags & omp::OpenMPOffloadMappingFlags::OMP_MAP_ATTACH))
12820 return;
12821
12822 // Reset the placeholder value to prepare the flag for the assignment of the
12823 // proper MEMBER_OF value.
12824 Flags &= ~omp::OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF;
12825 Flags |= MemberOfFlag;
12826}
12827
12828Constant *OpenMPIRBuilder::getAddrOfDeclareTargetVar(
12829 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind CaptureClause,
12830 OffloadEntriesInfoManager::OMPTargetDeviceClauseKind DeviceClause,
12831 bool IsDeclaration, bool IsExternallyVisible,
12832 TargetRegionEntryInfo EntryInfo, StringRef MangledName,
12833 std::vector<GlobalVariable *> &GeneratedRefs, bool OpenMPSIMD,
12834 std::vector<Triple> TargetTriple, Type *LlvmPtrTy,
12835 std::function<Constant *()> GlobalInitializer,
12836 std::function<GlobalValue::LinkageTypes()> VariableLinkage) {
12837 // TODO: convert this to utilise the IRBuilder Config rather than
12838 // a passed down argument.
12839 if (OpenMPSIMD)
12840 return nullptr;
12841
12842 if (CaptureClause == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink ||
12843 ((CaptureClause == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo ||
12844 CaptureClause ==
12845 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter) &&
12846 Config.hasRequiresUnifiedSharedMemory())) {
12847 SmallString<64> PtrName;
12848 {
12849 raw_svector_ostream OS(PtrName);
12850 OS << MangledName;
12851 if (!IsExternallyVisible)
12852 OS << format(Fmt: "_%x", Vals: EntryInfo.FileID);
12853 OS << "_decl_tgt_ref_ptr";
12854 }
12855
12856 Value *Ptr = M.getNamedValue(Name: PtrName);
12857
12858 if (!Ptr) {
12859 GlobalValue *GlobalValue = M.getNamedValue(Name: MangledName);
12860 Ptr = getOrCreateInternalVariable(Ty: LlvmPtrTy, Name: PtrName);
12861
12862 auto *GV = cast<GlobalVariable>(Val: Ptr);
12863 GV->setLinkage(GlobalValue::WeakAnyLinkage);
12864
12865 if (!Config.isTargetDevice()) {
12866 if (GlobalInitializer)
12867 GV->setInitializer(GlobalInitializer());
12868 else
12869 GV->setInitializer(GlobalValue);
12870 }
12871
12872 registerTargetGlobalVariable(
12873 CaptureClause, DeviceClause, IsDeclaration, IsExternallyVisible,
12874 EntryInfo, MangledName, GeneratedRefs, OpenMPSIMD, TargetTriple,
12875 GlobalInitializer, VariableLinkage, LlvmPtrTy, Addr: cast<Constant>(Val: Ptr));
12876 }
12877
12878 return cast<Constant>(Val: Ptr);
12879 }
12880
12881 return nullptr;
12882}
12883
12884void OpenMPIRBuilder::registerTargetGlobalVariable(
12885 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind CaptureClause,
12886 OffloadEntriesInfoManager::OMPTargetDeviceClauseKind DeviceClause,
12887 bool IsDeclaration, bool IsExternallyVisible,
12888 TargetRegionEntryInfo EntryInfo, StringRef MangledName,
12889 std::vector<GlobalVariable *> &GeneratedRefs, bool OpenMPSIMD,
12890 std::vector<Triple> TargetTriple,
12891 std::function<Constant *()> GlobalInitializer,
12892 std::function<GlobalValue::LinkageTypes()> VariableLinkage, Type *LlvmPtrTy,
12893 Constant *Addr) {
12894 if (DeviceClause != OffloadEntriesInfoManager::OMPTargetDeviceClauseAny ||
12895 (TargetTriple.empty() && !Config.isTargetDevice()))
12896 return;
12897
12898 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind Flags;
12899 StringRef VarName;
12900 int64_t VarSize;
12901 GlobalValue::LinkageTypes Linkage;
12902
12903 if ((CaptureClause == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo ||
12904 CaptureClause ==
12905 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter) &&
12906 !Config.hasRequiresUnifiedSharedMemory()) {
12907 Flags = OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo;
12908 VarName = MangledName;
12909 GlobalValue *LlvmVal = M.getNamedValue(Name: VarName);
12910
12911 if (!IsDeclaration)
12912 VarSize = divideCeil(
12913 Numerator: M.getDataLayout().getTypeSizeInBits(Ty: LlvmVal->getValueType()), Denominator: 8);
12914 else
12915 VarSize = 0;
12916 Linkage = (VariableLinkage) ? VariableLinkage() : LlvmVal->getLinkage();
12917
12918 // This is a workaround carried over from Clang which prevents undesired
12919 // optimisation of internal variables.
12920 if (Config.isTargetDevice() &&
12921 (!IsExternallyVisible || Linkage == GlobalValue::LinkOnceODRLinkage)) {
12922 // Do not create a "ref-variable" if the original is not also available
12923 // on the host.
12924 if (!OffloadInfoManager.hasDeviceGlobalVarEntryInfo(VarName))
12925 return;
12926
12927 std::string RefName = createPlatformSpecificName(Parts: {VarName, "ref"});
12928
12929 if (!M.getNamedValue(Name: RefName)) {
12930 Constant *AddrRef =
12931 getOrCreateInternalVariable(Ty: Addr->getType(), Name: RefName);
12932 auto *GvAddrRef = cast<GlobalVariable>(Val: AddrRef);
12933 GvAddrRef->setConstant(true);
12934 GvAddrRef->setLinkage(GlobalValue::InternalLinkage);
12935 GvAddrRef->setInitializer(Addr);
12936 GeneratedRefs.push_back(x: GvAddrRef);
12937 }
12938 }
12939 } else {
12940 if (CaptureClause == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink)
12941 Flags = OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink;
12942 else
12943 Flags = OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo;
12944
12945 if (Config.isTargetDevice()) {
12946 VarName = (Addr) ? Addr->getName() : "";
12947 Addr = nullptr;
12948 } else {
12949 Addr = getAddrOfDeclareTargetVar(
12950 CaptureClause, DeviceClause, IsDeclaration, IsExternallyVisible,
12951 EntryInfo, MangledName, GeneratedRefs, OpenMPSIMD, TargetTriple,
12952 LlvmPtrTy, GlobalInitializer, VariableLinkage);
12953 VarName = (Addr) ? Addr->getName() : "";
12954 }
12955 VarSize = M.getDataLayout().getPointerSize();
12956 Linkage = GlobalValue::WeakAnyLinkage;
12957 }
12958
12959 OffloadInfoManager.registerDeviceGlobalVarEntryInfo(VarName, Addr, VarSize,
12960 Flags, Linkage);
12961}
12962
12963/// Loads all the offload entries information from the host IR
12964/// metadata.
12965void OpenMPIRBuilder::loadOffloadInfoMetadata(Module &M) {
12966 // If we are in target mode, load the metadata from the host IR. This code has
12967 // to match the metadata creation in createOffloadEntriesAndInfoMetadata().
12968
12969 NamedMDNode *MD = M.getNamedMetadata(Name: ompOffloadInfoName);
12970 if (!MD)
12971 return;
12972
12973 for (MDNode *MN : MD->operands()) {
12974 auto &&GetMDInt = [MN](unsigned Idx) {
12975 auto *V = cast<ConstantAsMetadata>(Val: MN->getOperand(I: Idx));
12976 return cast<ConstantInt>(Val: V->getValue())->getZExtValue();
12977 };
12978
12979 auto &&GetMDString = [MN](unsigned Idx) {
12980 auto *V = cast<MDString>(Val: MN->getOperand(I: Idx));
12981 return V->getString();
12982 };
12983
12984 switch (GetMDInt(0)) {
12985 default:
12986 llvm_unreachable("Unexpected metadata!");
12987 break;
12988 case OffloadEntriesInfoManager::OffloadEntryInfo::
12989 OffloadingEntryInfoTargetRegion: {
12990 TargetRegionEntryInfo EntryInfo(/*ParentName=*/GetMDString(3),
12991 /*DeviceID=*/GetMDInt(1),
12992 /*FileID=*/GetMDInt(2),
12993 /*Line=*/GetMDInt(4),
12994 /*Count=*/GetMDInt(5));
12995 OffloadInfoManager.initializeTargetRegionEntryInfo(EntryInfo,
12996 /*Order=*/GetMDInt(6));
12997 break;
12998 }
12999 case OffloadEntriesInfoManager::OffloadEntryInfo::
13000 OffloadingEntryInfoDeviceGlobalVar:
13001 OffloadInfoManager.initializeDeviceGlobalVarEntryInfo(
13002 /*MangledName=*/Name: GetMDString(1),
13003 Flags: static_cast<OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind>(
13004 /*Flags=*/GetMDInt(2)),
13005 /*Order=*/GetMDInt(3));
13006 break;
13007 }
13008 }
13009}
13010
13011void OpenMPIRBuilder::loadOffloadInfoMetadata(vfs::FileSystem &VFS,
13012 StringRef HostFilePath) {
13013 if (HostFilePath.empty())
13014 return;
13015
13016 auto Buf = VFS.getBufferForFile(Name: HostFilePath);
13017 if (std::error_code Err = Buf.getError()) {
13018 report_fatal_error(reason: ("error opening host file from host file path inside of "
13019 "OpenMPIRBuilder: " +
13020 Err.message())
13021 .c_str());
13022 }
13023
13024 LLVMContext Ctx;
13025 auto M = expectedToErrorOrAndEmitErrors(
13026 Ctx, Val: parseBitcodeFile(Buffer: Buf.get()->getMemBufferRef(), Context&: Ctx));
13027 if (std::error_code Err = M.getError()) {
13028 report_fatal_error(
13029 reason: ("error parsing host file inside of OpenMPIRBuilder: " + Err.message())
13030 .c_str());
13031 }
13032
13033 loadOffloadInfoMetadata(M&: *M.get());
13034}
13035
13036OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createIteratorLoop(
13037 LocationDescription Loc, llvm::Value *TripCount, IteratorBodyGenTy BodyGen,
13038 llvm::StringRef Name) {
13039 Builder.restoreIP(IP: Loc.IP);
13040
13041 BasicBlock *CurBB = Builder.GetInsertBlock();
13042 assert(CurBB &&
13043 "expected a valid insertion block for creating an iterator loop");
13044 Function *F = CurBB->getParent();
13045
13046 InsertPointTy SplitIP = Builder.saveIP();
13047 if (SplitIP == CurBB->end())
13048 if (Instruction *Terminator = CurBB->getTerminatorOrNull())
13049 SplitIP = Terminator->getIterator();
13050
13051 BasicBlock *ContBB =
13052 splitBB(IP: SplitIP, /*CreateBranch=*/false,
13053 DL: Builder.getCurrentDebugLocation(), Name: "omp.it.cont");
13054
13055 CanonicalLoopInfo *CLI =
13056 createLoopSkeleton(DL: Builder.getCurrentDebugLocation(), TripCount, F,
13057 /*PreInsertBefore=*/ContBB,
13058 /*PostInsertBefore=*/ContBB, Name);
13059
13060 // Enter loop from original block.
13061 redirectTo(Source: CurBB, Target: CLI->getPreheader(), DL: Builder.getCurrentDebugLocation());
13062
13063 // Remove the unconditional branch inserted by createLoopSkeleton in the body
13064 if (Instruction *T = CLI->getBody()->getTerminatorOrNull())
13065 T->eraseFromParent();
13066
13067 InsertPointTy BodyIP = CLI->getBodyIP();
13068 // Blocks that exist before BodyGen, other than the body and latch, are
13069 // outside the loop body.
13070 SmallPtrSet<BasicBlock *, 32> ExistingBlocks;
13071 for (BasicBlock &Block : *F)
13072 ExistingBlocks.insert(Ptr: &Block);
13073 if (llvm::Error Err = BodyGen(BodyIP, CLI->getIndVar()))
13074 return Err;
13075
13076 // The body may span several blocks. Branch its single unterminated block to
13077 // the latch; otherwise some block must already branch there.
13078 BasicBlock *Latch = CLI->getLatch();
13079 BasicBlock *OpenBB = nullptr;
13080 bool ReachesLatch = false;
13081 SmallVector<BasicBlock *> Worklist{CLI->getBody()};
13082 SmallPtrSet<BasicBlock *, 8> Visited{CLI->getBody()};
13083 while (!Worklist.empty()) {
13084 BasicBlock *BB = Worklist.pop_back_val();
13085 if (!BB->hasTerminator()) {
13086 if (OpenBB)
13087 return make_error<StringError>(
13088 Args: "iterator bodygen must leave at most one unterminated block",
13089 Args: inconvertibleErrorCode());
13090 OpenBB = BB;
13091 continue;
13092 }
13093 for (BasicBlock *Succ : successors(BB)) {
13094 if (Succ == Latch) {
13095 ReachesLatch = true;
13096 continue;
13097 }
13098 if (Succ != CLI->getBody() && ExistingBlocks.contains(Ptr: Succ))
13099 return make_error<StringError>(
13100 Args: "iterator bodygen must not branch out of the loop body",
13101 Args: inconvertibleErrorCode());
13102 if (Visited.insert(Ptr: Succ).second)
13103 Worklist.push_back(Elt: Succ);
13104 }
13105 }
13106
13107 if (OpenBB) {
13108 Builder.SetInsertPoint(OpenBB);
13109 Builder.CreateBr(Dest: Latch);
13110 } else if (!ReachesLatch) {
13111 return make_error<StringError>(Args: "iterator bodygen must reach the loop latch",
13112 Args: inconvertibleErrorCode());
13113 }
13114
13115 // Link After -> ContBB
13116 Builder.SetInsertPoint(CLI->getAfter()->begin());
13117 if (!CLI->getAfter()->hasTerminator())
13118 Builder.CreateBr(Dest: ContBB);
13119
13120 return ContBB->begin();
13121}
13122
13123/// Mangle the parameter part of the vector function name according to
13124/// their OpenMP classification. The mangling function is defined in
13125/// section 4.5 of the AAVFABI(2021Q1).
13126static std::string mangleVectorParameters(
13127 ArrayRef<llvm::OpenMPIRBuilder::DeclareSimdAttrTy> ParamAttrs) {
13128 SmallString<256> Buffer;
13129 llvm::raw_svector_ostream Out(Buffer);
13130 for (const auto &ParamAttr : ParamAttrs) {
13131 switch (ParamAttr.Kind) {
13132 case llvm::OpenMPIRBuilder::DeclareSimdKindTy::Linear:
13133 Out << 'l';
13134 break;
13135 case llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearRef:
13136 Out << 'R';
13137 break;
13138 case llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearUVal:
13139 Out << 'U';
13140 break;
13141 case llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearVal:
13142 Out << 'L';
13143 break;
13144 case llvm::OpenMPIRBuilder::DeclareSimdKindTy::Uniform:
13145 Out << 'u';
13146 break;
13147 case llvm::OpenMPIRBuilder::DeclareSimdKindTy::Vector:
13148 Out << 'v';
13149 break;
13150 }
13151 if (ParamAttr.HasVarStride)
13152 Out << "s" << ParamAttr.StrideOrArg;
13153 else if (ParamAttr.Kind ==
13154 llvm::OpenMPIRBuilder::DeclareSimdKindTy::Linear ||
13155 ParamAttr.Kind ==
13156 llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearRef ||
13157 ParamAttr.Kind ==
13158 llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearUVal ||
13159 ParamAttr.Kind ==
13160 llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearVal) {
13161 // Don't print the step value if it is not present or if it is
13162 // equal to 1.
13163 if (ParamAttr.StrideOrArg < 0)
13164 Out << 'n' << -ParamAttr.StrideOrArg;
13165 else if (ParamAttr.StrideOrArg != 1)
13166 Out << ParamAttr.StrideOrArg;
13167 }
13168
13169 if (!!ParamAttr.Alignment)
13170 Out << 'a' << ParamAttr.Alignment;
13171 }
13172
13173 return std::string(Out.str());
13174}
13175
13176void OpenMPIRBuilder::emitX86DeclareSimdFunction(
13177 llvm::Function *Fn, unsigned NumElts, const llvm::APSInt &VLENVal,
13178 llvm::ArrayRef<DeclareSimdAttrTy> ParamAttrs, DeclareSimdBranch Branch) {
13179 struct ISADataTy {
13180 char ISA;
13181 unsigned VecRegSize;
13182 };
13183 ISADataTy ISAData[] = {
13184 {.ISA: 'b', .VecRegSize: 128}, // SSE
13185 {.ISA: 'c', .VecRegSize: 256}, // AVX
13186 {.ISA: 'd', .VecRegSize: 256}, // AVX2
13187 {.ISA: 'e', .VecRegSize: 512}, // AVX512
13188 };
13189 llvm::SmallVector<char, 2> Masked;
13190 switch (Branch) {
13191 case DeclareSimdBranch::Undefined:
13192 Masked.push_back(Elt: 'N');
13193 Masked.push_back(Elt: 'M');
13194 break;
13195 case DeclareSimdBranch::Notinbranch:
13196 Masked.push_back(Elt: 'N');
13197 break;
13198 case DeclareSimdBranch::Inbranch:
13199 Masked.push_back(Elt: 'M');
13200 break;
13201 }
13202 for (char Mask : Masked) {
13203 for (const ISADataTy &Data : ISAData) {
13204 llvm::SmallString<256> Buffer;
13205 llvm::raw_svector_ostream Out(Buffer);
13206 Out << "_ZGV" << Data.ISA << Mask;
13207 if (!VLENVal) {
13208 assert(NumElts && "Non-zero simdlen/cdtsize expected");
13209 Out << llvm::APSInt::getUnsigned(X: Data.VecRegSize / NumElts);
13210 } else {
13211 Out << VLENVal;
13212 }
13213 Out << mangleVectorParameters(ParamAttrs);
13214 Out << '_' << Fn->getName();
13215 Fn->addFnAttr(Kind: Out.str());
13216 }
13217 }
13218}
13219
13220// Function used to add the attribute. The parameter `VLEN` is templated to
13221// allow the use of `x` when targeting scalable functions for SVE.
13222template <typename T>
13223static void addAArch64VectorName(T VLEN, StringRef LMask, StringRef Prefix,
13224 char ISA, StringRef ParSeq,
13225 StringRef MangledName, bool OutputBecomesInput,
13226 llvm::Function *Fn) {
13227 SmallString<256> Buffer;
13228 llvm::raw_svector_ostream Out(Buffer);
13229 Out << Prefix << ISA << LMask << VLEN;
13230 if (OutputBecomesInput)
13231 Out << 'v';
13232 Out << ParSeq << '_' << MangledName;
13233 Fn->addFnAttr(Kind: Out.str());
13234}
13235
13236// Helper function to generate the Advanced SIMD names depending on the value
13237// of the NDS when simdlen is not present.
13238static void addAArch64AdvSIMDNDSNames(unsigned NDS, StringRef Mask,
13239 StringRef Prefix, char ISA,
13240 StringRef ParSeq, StringRef MangledName,
13241 bool OutputBecomesInput,
13242 llvm::Function *Fn) {
13243 switch (NDS) {
13244 case 8:
13245 addAArch64VectorName(VLEN: 8, LMask: Mask, Prefix, ISA, ParSeq, MangledName,
13246 OutputBecomesInput, Fn);
13247 addAArch64VectorName(VLEN: 16, LMask: Mask, Prefix, ISA, ParSeq, MangledName,
13248 OutputBecomesInput, Fn);
13249 break;
13250 case 16:
13251 addAArch64VectorName(VLEN: 4, LMask: Mask, Prefix, ISA, ParSeq, MangledName,
13252 OutputBecomesInput, Fn);
13253 addAArch64VectorName(VLEN: 8, LMask: Mask, Prefix, ISA, ParSeq, MangledName,
13254 OutputBecomesInput, Fn);
13255 break;
13256 case 32:
13257 addAArch64VectorName(VLEN: 2, LMask: Mask, Prefix, ISA, ParSeq, MangledName,
13258 OutputBecomesInput, Fn);
13259 addAArch64VectorName(VLEN: 4, LMask: Mask, Prefix, ISA, ParSeq, MangledName,
13260 OutputBecomesInput, Fn);
13261 break;
13262 case 64:
13263 case 128:
13264 addAArch64VectorName(VLEN: 2, LMask: Mask, Prefix, ISA, ParSeq, MangledName,
13265 OutputBecomesInput, Fn);
13266 break;
13267 default:
13268 llvm_unreachable("Scalar type is too wide.");
13269 }
13270}
13271
13272/// Emit vector function attributes for AArch64, as defined in the AAVFABI.
13273void OpenMPIRBuilder::emitAArch64DeclareSimdFunction(
13274 llvm::Function *Fn, unsigned UserVLEN,
13275 llvm::ArrayRef<DeclareSimdAttrTy> ParamAttrs, DeclareSimdBranch Branch,
13276 char ISA, unsigned NarrowestDataSize, bool OutputBecomesInput) {
13277 assert((ISA == 'n' || ISA == 's') && "Expected ISA either 's' or 'n'.");
13278
13279 // Sort out parameter sequence.
13280 const std::string ParSeq = mangleVectorParameters(ParamAttrs);
13281 StringRef Prefix = "_ZGV";
13282 StringRef MangledName = Fn->getName();
13283
13284 // Generate simdlen from user input (if any).
13285 if (UserVLEN) {
13286 if (ISA == 's') {
13287 // SVE generates only a masked function.
13288 addAArch64VectorName(VLEN: UserVLEN, LMask: "M", Prefix, ISA, ParSeq, MangledName,
13289 OutputBecomesInput, Fn);
13290 return;
13291 }
13292
13293 switch (Branch) {
13294 case DeclareSimdBranch::Undefined:
13295 addAArch64VectorName(VLEN: UserVLEN, LMask: "N", Prefix, ISA, ParSeq, MangledName,
13296 OutputBecomesInput, Fn);
13297 addAArch64VectorName(VLEN: UserVLEN, LMask: "M", Prefix, ISA, ParSeq, MangledName,
13298 OutputBecomesInput, Fn);
13299 break;
13300 case DeclareSimdBranch::Inbranch:
13301 addAArch64VectorName(VLEN: UserVLEN, LMask: "M", Prefix, ISA, ParSeq, MangledName,
13302 OutputBecomesInput, Fn);
13303 break;
13304 case DeclareSimdBranch::Notinbranch:
13305 addAArch64VectorName(VLEN: UserVLEN, LMask: "N", Prefix, ISA, ParSeq, MangledName,
13306 OutputBecomesInput, Fn);
13307 break;
13308 }
13309 return;
13310 }
13311
13312 if (ISA == 's') {
13313 // SVE, section 3.4.1, item 1.
13314 addAArch64VectorName(VLEN: "x", LMask: "M", Prefix, ISA, ParSeq, MangledName,
13315 OutputBecomesInput, Fn);
13316 return;
13317 }
13318
13319 switch (Branch) {
13320 case DeclareSimdBranch::Undefined:
13321 addAArch64AdvSIMDNDSNames(NDS: NarrowestDataSize, Mask: "N", Prefix, ISA, ParSeq,
13322 MangledName, OutputBecomesInput, Fn);
13323 addAArch64AdvSIMDNDSNames(NDS: NarrowestDataSize, Mask: "M", Prefix, ISA, ParSeq,
13324 MangledName, OutputBecomesInput, Fn);
13325 break;
13326 case DeclareSimdBranch::Inbranch:
13327 addAArch64AdvSIMDNDSNames(NDS: NarrowestDataSize, Mask: "M", Prefix, ISA, ParSeq,
13328 MangledName, OutputBecomesInput, Fn);
13329 break;
13330 case DeclareSimdBranch::Notinbranch:
13331 addAArch64AdvSIMDNDSNames(NDS: NarrowestDataSize, Mask: "N", Prefix, ISA, ParSeq,
13332 MangledName, OutputBecomesInput, Fn);
13333 break;
13334 }
13335}
13336
13337//===----------------------------------------------------------------------===//
13338// OffloadEntriesInfoManager
13339//===----------------------------------------------------------------------===//
13340
13341bool OffloadEntriesInfoManager::empty() const {
13342 return OffloadEntriesTargetRegion.empty() &&
13343 OffloadEntriesDeviceGlobalVar.empty();
13344}
13345
13346unsigned OffloadEntriesInfoManager::getTargetRegionEntryInfoCount(
13347 const TargetRegionEntryInfo &EntryInfo) const {
13348 auto It = OffloadEntriesTargetRegionCount.find(
13349 x: getTargetRegionEntryCountKey(EntryInfo));
13350 if (It == OffloadEntriesTargetRegionCount.end())
13351 return 0;
13352 return It->second;
13353}
13354
13355void OffloadEntriesInfoManager::incrementTargetRegionEntryInfoCount(
13356 const TargetRegionEntryInfo &EntryInfo) {
13357 OffloadEntriesTargetRegionCount[getTargetRegionEntryCountKey(EntryInfo)] =
13358 EntryInfo.Count + 1;
13359}
13360
13361/// Initialize target region entry.
13362void OffloadEntriesInfoManager::initializeTargetRegionEntryInfo(
13363 const TargetRegionEntryInfo &EntryInfo, unsigned Order) {
13364 OffloadEntriesTargetRegion[EntryInfo] =
13365 OffloadEntryInfoTargetRegion(Order, /*Addr=*/nullptr, /*ID=*/nullptr,
13366 OMPTargetRegionEntryTargetRegion);
13367 ++OffloadingEntriesNum;
13368}
13369
13370void OffloadEntriesInfoManager::registerTargetRegionEntryInfo(
13371 TargetRegionEntryInfo EntryInfo, Constant *Addr, Constant *ID,
13372 OMPTargetRegionEntryKind Flags) {
13373 assert(EntryInfo.Count == 0 && "expected default EntryInfo");
13374
13375 // Update the EntryInfo with the next available count for this location.
13376 EntryInfo.Count = getTargetRegionEntryInfoCount(EntryInfo);
13377
13378 // If we are emitting code for a target, the entry is already initialized,
13379 // only has to be registered.
13380 if (OMPBuilder->Config.isTargetDevice()) {
13381 // This could happen if the device compilation is invoked standalone.
13382 if (!hasTargetRegionEntryInfo(EntryInfo)) {
13383 return;
13384 }
13385 auto &Entry = OffloadEntriesTargetRegion[EntryInfo];
13386 Entry.setAddress(Addr);
13387 Entry.setID(ID);
13388 Entry.setFlags(Flags);
13389 } else {
13390 if (Flags == OffloadEntriesInfoManager::OMPTargetRegionEntryTargetRegion &&
13391 hasTargetRegionEntryInfo(EntryInfo, /*IgnoreAddressId*/ true))
13392 return;
13393 assert(!hasTargetRegionEntryInfo(EntryInfo) &&
13394 "Target region entry already registered!");
13395 OffloadEntryInfoTargetRegion Entry(OffloadingEntriesNum, Addr, ID, Flags);
13396 OffloadEntriesTargetRegion[EntryInfo] = Entry;
13397 ++OffloadingEntriesNum;
13398 }
13399 incrementTargetRegionEntryInfoCount(EntryInfo);
13400}
13401
13402bool OffloadEntriesInfoManager::hasTargetRegionEntryInfo(
13403 TargetRegionEntryInfo EntryInfo, bool IgnoreAddressId) const {
13404
13405 // Update the EntryInfo with the next available count for this location.
13406 EntryInfo.Count = getTargetRegionEntryInfoCount(EntryInfo);
13407
13408 auto It = OffloadEntriesTargetRegion.find(x: EntryInfo);
13409 if (It == OffloadEntriesTargetRegion.end()) {
13410 return false;
13411 }
13412 // Fail if this entry is already registered.
13413 if (!IgnoreAddressId && (It->second.getAddress() || It->second.getID()))
13414 return false;
13415 return true;
13416}
13417
13418void OffloadEntriesInfoManager::actOnTargetRegionEntriesInfo(
13419 const OffloadTargetRegionEntryInfoActTy &Action) {
13420 // Scan all target region entries and perform the provided action.
13421 for (const auto &It : OffloadEntriesTargetRegion) {
13422 Action(It.first, It.second);
13423 }
13424}
13425
13426void OffloadEntriesInfoManager::initializeDeviceGlobalVarEntryInfo(
13427 StringRef Name, OMPTargetGlobalVarEntryKind Flags, unsigned Order) {
13428 OffloadEntriesDeviceGlobalVar.try_emplace(Key: Name, Args&: Order, Args&: Flags);
13429 ++OffloadingEntriesNum;
13430}
13431
13432void OffloadEntriesInfoManager::registerDeviceGlobalVarEntryInfo(
13433 StringRef VarName, Constant *Addr, int64_t VarSize,
13434 OMPTargetGlobalVarEntryKind Flags, GlobalValue::LinkageTypes Linkage) {
13435 if (OMPBuilder->Config.isTargetDevice()) {
13436 // This could happen if the device compilation is invoked standalone.
13437 if (!hasDeviceGlobalVarEntryInfo(VarName))
13438 return;
13439 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
13440 if (Entry.getAddress() && hasDeviceGlobalVarEntryInfo(VarName)) {
13441 if (Entry.getVarSize() == 0) {
13442 Entry.setVarSize(VarSize);
13443 Entry.setLinkage(Linkage);
13444 }
13445 return;
13446 }
13447 Entry.setVarSize(VarSize);
13448 Entry.setLinkage(Linkage);
13449 Entry.setAddress(Addr);
13450 } else {
13451 if (hasDeviceGlobalVarEntryInfo(VarName)) {
13452 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
13453 assert(Entry.isValid() && Entry.getFlags() == Flags &&
13454 "Entry not initialized!");
13455 if (Entry.getVarSize() == 0) {
13456 Entry.setVarSize(VarSize);
13457 Entry.setLinkage(Linkage);
13458 }
13459 return;
13460 }
13461 if (Flags == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect ||
13462 Flags ==
13463 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirectVTable)
13464 OffloadEntriesDeviceGlobalVar.try_emplace(Key: VarName, Args&: OffloadingEntriesNum,
13465 Args&: Addr, Args&: VarSize, Args&: Flags, Args&: Linkage,
13466 Args: VarName.str());
13467 else
13468 OffloadEntriesDeviceGlobalVar.try_emplace(
13469 Key: VarName, Args&: OffloadingEntriesNum, Args&: Addr, Args&: VarSize, Args&: Flags, Args&: Linkage, Args: "");
13470 ++OffloadingEntriesNum;
13471 }
13472}
13473
13474void OffloadEntriesInfoManager::actOnDeviceGlobalVarEntriesInfo(
13475 const OffloadDeviceGlobalVarEntryInfoActTy &Action) {
13476 // Scan all target region entries and perform the provided action.
13477 for (const auto &E : OffloadEntriesDeviceGlobalVar)
13478 Action(E.getKey(), E.getValue());
13479}
13480
13481//===----------------------------------------------------------------------===//
13482// CanonicalLoopInfo
13483//===----------------------------------------------------------------------===//
13484
13485void CanonicalLoopInfo::reportUnsupportedVirtualCollapse() const {
13486 assert(isValid() && "Requires a valid canonical loop");
13487 assert(VirtualCollapse && "Requires Virtual collapse state");
13488 reportFatalInternalError(
13489 reason: "virtual collapse must be the final structural loop transformation");
13490}
13491
13492void CanonicalLoopInfo::collectControlBlocks(
13493 SmallVectorImpl<BasicBlock *> &BBs) {
13494 if (VirtualCollapse)
13495 reportUnsupportedVirtualCollapse();
13496 // We only count those BBs as control block for which we do not need to
13497 // reverse the CFG, i.e. not the loop body which can contain arbitrary control
13498 // flow. For consistency, this also means we do not add the Body block, which
13499 // is just the entry to the body code.
13500 BBs.reserve(N: BBs.size() + 6);
13501 BBs.append(IL: {getPreheader(), Header, Cond, Latch, Exit, getAfter()});
13502}
13503
13504BasicBlock *CanonicalLoopInfo::getVirtualCollapseInit() const {
13505 return VirtualCollapse ? VirtualCollapse->Count->getParent() : nullptr;
13506}
13507
13508BasicBlock *CanonicalLoopInfo::getPreheader() const {
13509 assert(isValid() && "Requires a valid canonical loop");
13510 if (VirtualCollapse)
13511 return getVirtualCollapseInit()->getSinglePredecessor();
13512
13513 for (BasicBlock *Pred : predecessors(BB: Header)) {
13514 if (Pred != Latch)
13515 return Pred;
13516 }
13517 llvm_unreachable("Missing preheader");
13518}
13519
13520void CanonicalLoopInfo::setTripCount(Value *TripCount) {
13521 assert(isValid() && "Requires a valid canonical loop");
13522
13523 Instruction *CmpI = &getCond()->front();
13524 assert(isa<CmpInst>(CmpI) && "First inst must compare IV with TripCount");
13525 CmpI->setOperand(i: 1, Val: TripCount);
13526
13527#ifndef NDEBUG
13528 assertOK();
13529#endif
13530}
13531
13532void CanonicalLoopInfo::mapIndVar(
13533 llvm::function_ref<Value *(Instruction *)> Updater) {
13534 assert(isValid() && "Requires a valid canonical loop");
13535
13536 Instruction *OldIV = getIndVar();
13537
13538 // Record all uses excluding those introduced by the updater. Uses by the
13539 // CanonicalLoopInfo itself to keep track of the number of iterations are
13540 // excluded.
13541 SmallVector<Use *> ReplacableUses;
13542 for (Use &U : OldIV->uses()) {
13543 auto *User = dyn_cast<Instruction>(Val: U.getUser());
13544 if (!User)
13545 continue;
13546 if (User->getParent() == getCond())
13547 continue;
13548 if (User->getParent() == getLatch())
13549 continue;
13550 ReplacableUses.push_back(Elt: &U);
13551 }
13552
13553 // Run the updater that may introduce new uses
13554 Value *NewIV = Updater(OldIV);
13555
13556 // Replace the old uses with the value returned by the updater.
13557 for (Use *U : ReplacableUses)
13558 U->set(NewIV);
13559
13560#ifndef NDEBUG
13561 assertOK();
13562#endif
13563}
13564
13565void CanonicalLoopInfo::assertOK() const {
13566#ifndef NDEBUG
13567 // No constraints if this object currently does not describe a loop.
13568 if (!isValid())
13569 return;
13570
13571 bool IsVirtual = VirtualCollapse.has_value();
13572 BasicBlock *Preheader = getPreheader();
13573 BasicBlock *Init = getVirtualCollapseInit();
13574 BasicBlock *IndVarEntry = Preheader;
13575 if (IsVirtual) {
13576 auto *Guard = cast<CondBrInst>(Init->getTerminator());
13577 IndVarEntry = Guard->getSuccessor(0);
13578 assert(IndVarEntry->getSinglePredecessor() == Init);
13579 assert(IndVarEntry->getSingleSuccessor() == Header);
13580 assert(Guard->getSuccessor(1) == Exit);
13581 assert(VirtualCollapse->Begin->getParent() == Init);
13582 for (BinaryOperator *Copy :
13583 {VirtualCollapse->Begin, VirtualCollapse->Count}) {
13584 auto *Zero = dyn_cast<ConstantInt>(Copy->getOperand(1));
13585 assert(Copy->getOpcode() == Instruction::Add && Zero && Zero->isZero() &&
13586 "Virtual range copies must add zero");
13587 }
13588 auto *NonEmpty = cast<ICmpInst>(Guard->getCondition());
13589 assert(NonEmpty->getPredicate() == ICmpInst::ICMP_NE &&
13590 NonEmpty->getOperand(0) == VirtualCollapse->Count &&
13591 cast<ConstantInt>(NonEmpty->getOperand(1))->isZero() &&
13592 "Collapsed guard must skip empty ranges with zero count");
13593 }
13594
13595 // Validate the retained physical loop without exposing a logical body or IV.
13596 BasicBlock *Body =
13597 IsVirtual ? cast<CondBrInst>(Cond->getTerminator())->getSuccessor(0)
13598 : getBody();
13599 BasicBlock *After = getAfter();
13600
13601 // Verify standard control-flow we use for OpenMP loops.
13602 assert(Preheader);
13603 assert(isa<UncondBrInst>(Preheader->getTerminator()) &&
13604 "Preheader must terminate with unconditional branch");
13605 if (IsVirtual)
13606 assert(Preheader->getSingleSuccessor() == Init &&
13607 "Preheader must jump to collapsed initialization");
13608 else
13609 assert(Preheader->getSingleSuccessor() == Header &&
13610 "Preheader must jump to header");
13611
13612 assert(Header);
13613 assert(isa<UncondBrInst>(Header->getTerminator()) &&
13614 "Header must terminate with unconditional branch");
13615 assert(Header->getSingleSuccessor() == Cond &&
13616 "Header must jump to exiting block");
13617
13618 assert(Cond);
13619 assert(Cond->getSinglePredecessor() == Header &&
13620 "Exiting block only reachable from header");
13621
13622 assert(isa<CondBrInst>(Cond->getTerminator()) &&
13623 "Exiting block must terminate with conditional branch");
13624 assert(cast<CondBrInst>(Cond->getTerminator())->getSuccessor(0) == Body &&
13625 "Exiting block's first successor jump to the body");
13626 assert(cast<CondBrInst>(Cond->getTerminator())->getSuccessor(1) == Exit &&
13627 "Exiting block's second successor must exit the loop");
13628
13629 assert(Body);
13630 assert(Body->getSinglePredecessor() == Cond &&
13631 "Body only reachable from exiting block");
13632 assert(!isa<PHINode>(Body->front()));
13633
13634 assert(Latch);
13635 assert(isa<UncondBrInst>(Latch->getTerminator()) &&
13636 "Latch must terminate with unconditional branch");
13637 assert(Latch->getSingleSuccessor() == Header && "Latch must jump to header");
13638 // TODO: To support simple redirecting of the end of the body code that has
13639 // multiple; introduce another auxiliary basic block like preheader and after.
13640 assert(Latch->getSinglePredecessor() != nullptr);
13641 assert(!isa<PHINode>(Latch->front()));
13642
13643 assert(Exit);
13644 assert(isa<UncondBrInst>(Exit->getTerminator()) &&
13645 "Exit block must terminate with unconditional branch");
13646 assert(Exit->getSingleSuccessor() == After &&
13647 "Exit block must jump to after block");
13648
13649 assert(After);
13650 assert(After->getSinglePredecessor() == Exit &&
13651 "After block only reachable from exit block");
13652 assert(After->empty() || !isa<PHINode>(After->front()));
13653
13654 Instruction *IndVar = IsVirtual ? &Header->front() : getIndVar();
13655 assert(IndVar && "Canonical induction variable not found?");
13656 assert(isa<IntegerType>(IndVar->getType()) &&
13657 "Induction variable must be an integer");
13658 assert(cast<PHINode>(IndVar)->getParent() == Header &&
13659 "Induction variable must be a PHI in the loop header");
13660 assert(cast<PHINode>(IndVar)->getIncomingBlock(0) == IndVarEntry);
13661 if (!IsVirtual)
13662 assert(cast<ConstantInt>(cast<PHINode>(IndVar)->getIncomingValue(0))
13663 ->isZero());
13664 assert(cast<PHINode>(IndVar)->getIncomingBlock(1) == Latch);
13665
13666 auto *NextIndVar = cast<PHINode>(IndVar)->getIncomingValue(1);
13667 assert(cast<Instruction>(NextIndVar)->getParent() == Latch);
13668 assert(cast<BinaryOperator>(NextIndVar)->getOpcode() == BinaryOperator::Add);
13669 assert(cast<BinaryOperator>(NextIndVar)->getOperand(0) == IndVar);
13670 assert(cast<ConstantInt>(cast<BinaryOperator>(NextIndVar)->getOperand(1))
13671 ->isOne());
13672
13673 Value *TripCount =
13674 IsVirtual ? cast<CmpInst>(&Cond->front())->getOperand(1) : getTripCount();
13675 assert(TripCount && "Loop trip count not found?");
13676 assert(IndVar->getType() == TripCount->getType() &&
13677 "Trip count and induction variable must have the same type");
13678
13679 auto *CmpI = cast<CmpInst>(&Cond->front());
13680 assert(CmpI->getPredicate() == CmpInst::ICMP_ULT &&
13681 "Exit condition must be a signed less-than comparison");
13682 assert(CmpI->getOperand(0) == IndVar &&
13683 "Exit condition must compare the induction variable");
13684 assert(CmpI->getOperand(1) == TripCount &&
13685 "Exit condition must compare with the trip count");
13686#endif
13687}
13688
13689void CanonicalLoopInfo::invalidate() {
13690 Header = nullptr;
13691 Cond = nullptr;
13692 Latch = nullptr;
13693 Exit = nullptr;
13694 VirtualCollapse.reset();
13695}
13696