1//===- OpenMPIRBuilder.cpp - Builder for LLVM-IR for OpenMP directives ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9///
10/// This file implements the OpenMPIRBuilder class, which is used as a
11/// convenient way to create LLVM instructions for OpenMP directives.
12///
13//===----------------------------------------------------------------------===//
14
15#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
16#include "llvm/ADT/SmallBitVector.h"
17#include "llvm/ADT/SmallSet.h"
18#include "llvm/ADT/SmallVectorExtras.h"
19#include "llvm/ADT/StringExtras.h"
20#include "llvm/ADT/StringRef.h"
21#include "llvm/Analysis/AssumptionCache.h"
22#include "llvm/Analysis/CodeMetrics.h"
23#include "llvm/Analysis/LoopInfo.h"
24#include "llvm/Analysis/OptimizationRemarkEmitter.h"
25#include "llvm/Analysis/PostDominators.h"
26#include "llvm/Analysis/ScalarEvolution.h"
27#include "llvm/Analysis/TargetLibraryInfo.h"
28#include "llvm/Bitcode/BitcodeReader.h"
29#include "llvm/Frontend/Offloading/Utility.h"
30#include "llvm/Frontend/OpenMP/OMPGridValues.h"
31#include "llvm/IR/Attributes.h"
32#include "llvm/IR/BasicBlock.h"
33#include "llvm/IR/CFG.h"
34#include "llvm/IR/CallingConv.h"
35#include "llvm/IR/Constant.h"
36#include "llvm/IR/Constants.h"
37#include "llvm/IR/DIBuilder.h"
38#include "llvm/IR/DebugInfoMetadata.h"
39#include "llvm/IR/DerivedTypes.h"
40#include "llvm/IR/Function.h"
41#include "llvm/IR/GlobalVariable.h"
42#include "llvm/IR/IRBuilder.h"
43#include "llvm/IR/InstIterator.h"
44#include "llvm/IR/IntrinsicInst.h"
45#include "llvm/IR/LLVMContext.h"
46#include "llvm/IR/MDBuilder.h"
47#include "llvm/IR/Metadata.h"
48#include "llvm/IR/PassInstrumentation.h"
49#include "llvm/IR/PassManager.h"
50#include "llvm/IR/ReplaceConstant.h"
51#include "llvm/IR/Value.h"
52#include "llvm/MC/TargetRegistry.h"
53#include "llvm/Support/CommandLine.h"
54#include "llvm/Support/Error.h"
55#include "llvm/Support/ErrorHandling.h"
56#include "llvm/Support/NVVMAttributes.h"
57#include "llvm/Support/VirtualFileSystem.h"
58#include "llvm/Target/TargetMachine.h"
59#include "llvm/Target/TargetOptions.h"
60#include "llvm/Transforms/Utils/BasicBlockUtils.h"
61#include "llvm/Transforms/Utils/Cloning.h"
62#include "llvm/Transforms/Utils/CodeExtractor.h"
63#include "llvm/Transforms/Utils/LoopPeel.h"
64#include "llvm/Transforms/Utils/UnrollLoop.h"
65
66#include <cstdint>
67#include <optional>
68
69#define DEBUG_TYPE "openmp-ir-builder"
70
71using namespace llvm;
72using namespace omp;
73
74static cl::opt<bool>
75 OptimisticAttributes("openmp-ir-builder-optimistic-attributes", cl::Hidden,
76 cl::desc("Use optimistic attributes describing "
77 "'as-if' properties of runtime calls."),
78 cl::init(Val: false));
79
80static cl::opt<double> UnrollThresholdFactor(
81 "openmp-ir-builder-unroll-threshold-factor", cl::Hidden,
82 cl::desc("Factor for the unroll threshold to account for code "
83 "simplifications still taking place"),
84 cl::init(Val: 1.5));
85
86static cl::opt<bool> UseDefaultMaxThreads(
87 "openmp-ir-builder-use-default-max-threads", cl::Hidden,
88 cl::desc("Use a default max threads if none is provided."), cl::init(Val: true));
89
90#ifndef NDEBUG
91/// Return whether IP1 and IP2 are ambiguous, i.e. that inserting instructions
92/// at position IP1 may change the meaning of IP2 or vice-versa. This is because
93/// an InsertPoint stores the instruction before something is inserted. For
94/// instance, if both point to the same instruction, two IRBuilders alternating
95/// creating instruction will cause the instructions to be interleaved.
96static bool isConflictIP(IRBuilder<>::InsertPoint IP1,
97 IRBuilder<>::InsertPoint IP2) {
98 if (!IP1.isSet() || !IP2.isSet())
99 return false;
100 return IP1.getBlock() == IP2.getBlock() && IP1.getPoint() == IP2.getPoint();
101}
102
103static bool isValidWorkshareLoopScheduleType(OMPScheduleType SchedType) {
104 // Valid ordered/unordered and base algorithm combinations.
105 switch (SchedType & ~OMPScheduleType::MonotonicityMask) {
106 case OMPScheduleType::UnorderedStaticChunked:
107 case OMPScheduleType::UnorderedStatic:
108 case OMPScheduleType::UnorderedDynamicChunked:
109 case OMPScheduleType::UnorderedGuidedChunked:
110 case OMPScheduleType::UnorderedRuntime:
111 case OMPScheduleType::UnorderedAuto:
112 case OMPScheduleType::UnorderedTrapezoidal:
113 case OMPScheduleType::UnorderedGreedy:
114 case OMPScheduleType::UnorderedBalanced:
115 case OMPScheduleType::UnorderedGuidedIterativeChunked:
116 case OMPScheduleType::UnorderedGuidedAnalyticalChunked:
117 case OMPScheduleType::UnorderedSteal:
118 case OMPScheduleType::UnorderedStaticBalancedChunked:
119 case OMPScheduleType::UnorderedGuidedSimd:
120 case OMPScheduleType::UnorderedRuntimeSimd:
121 case OMPScheduleType::OrderedStaticChunked:
122 case OMPScheduleType::OrderedStatic:
123 case OMPScheduleType::OrderedDynamicChunked:
124 case OMPScheduleType::OrderedGuidedChunked:
125 case OMPScheduleType::OrderedRuntime:
126 case OMPScheduleType::OrderedAuto:
127 case OMPScheduleType::OrderdTrapezoidal:
128 case OMPScheduleType::NomergeUnorderedStaticChunked:
129 case OMPScheduleType::NomergeUnorderedStatic:
130 case OMPScheduleType::NomergeUnorderedDynamicChunked:
131 case OMPScheduleType::NomergeUnorderedGuidedChunked:
132 case OMPScheduleType::NomergeUnorderedRuntime:
133 case OMPScheduleType::NomergeUnorderedAuto:
134 case OMPScheduleType::NomergeUnorderedTrapezoidal:
135 case OMPScheduleType::NomergeUnorderedGreedy:
136 case OMPScheduleType::NomergeUnorderedBalanced:
137 case OMPScheduleType::NomergeUnorderedGuidedIterativeChunked:
138 case OMPScheduleType::NomergeUnorderedGuidedAnalyticalChunked:
139 case OMPScheduleType::NomergeUnorderedSteal:
140 case OMPScheduleType::NomergeOrderedStaticChunked:
141 case OMPScheduleType::NomergeOrderedStatic:
142 case OMPScheduleType::NomergeOrderedDynamicChunked:
143 case OMPScheduleType::NomergeOrderedGuidedChunked:
144 case OMPScheduleType::NomergeOrderedRuntime:
145 case OMPScheduleType::NomergeOrderedAuto:
146 case OMPScheduleType::NomergeOrderedTrapezoidal:
147 case OMPScheduleType::OrderedDistributeChunked:
148 case OMPScheduleType::OrderedDistribute:
149 break;
150 default:
151 return false;
152 }
153
154 // Must not set both monotonicity modifiers at the same time.
155 OMPScheduleType MonotonicityFlags =
156 SchedType & OMPScheduleType::MonotonicityMask;
157 if (MonotonicityFlags == OMPScheduleType::MonotonicityMask)
158 return false;
159
160 return true;
161}
162#endif
163
164/// This is a wrapper over IRBuilderBase::restoreIP that also restores a current
165/// debug location when the insert point is at the end of a block. It picks a
166/// location scoped to the current function: the block's last instruction
167/// location if the block is non-empty, otherwise a location synthesized from
168/// the function's subprogram (when the function has debug info).
169static void restoreIPandDebugLoc(llvm::IRBuilderBase &Builder,
170 llvm::IRBuilderBase::InsertPoint IP) {
171 Builder.restoreIP(IP);
172 // When IP points at a real instruction, restoreIP (SetInsertPoint) already
173 // set the debug location from that instruction, so leave it alone.
174 llvm::BasicBlock *BB = Builder.GetInsertBlock();
175 if (Builder.GetInsertPoint() != BB->end())
176 return;
177
178 // At the end of a block, pick a location guaranteed to belong to the current
179 // insertion function's subprogram. Prefer the block's own last instruction;
180 // otherwise synthesize a location from the function's subprogram.
181 if (!BB->empty())
182 Builder.SetCurrentDebugLocation(BB->back().getStableDebugLoc());
183 else if (llvm::DISubprogram *FSP =
184 BB->getParent() ? BB->getParent()->getSubprogram() : nullptr) {
185 unsigned Line = FSP->getScopeLine() ? FSP->getScopeLine() : FSP->getLine();
186 Builder.SetCurrentDebugLocation(
187 llvm::DILocation::get(Context&: FSP->getContext(), Line, /*Column=*/0, Scope: FSP));
188 }
189}
190
191static bool hasGridValue(const Triple &T) {
192 return T.isAMDGPU() || T.isNVPTX() || T.isSPIRV();
193}
194
195static const omp::GV &getGridValue(const Triple &T, Function *Kernel) {
196 if (T.isAMDGPU()) {
197 StringRef Features =
198 Kernel->getFnAttribute(Kind: "target-features").getValueAsString();
199 if (Features.count(Str: "+wavefrontsize64"))
200 return omp::getAMDGPUGridValues<64>();
201 return omp::getAMDGPUGridValues<32>();
202 }
203 if (T.isNVPTX())
204 return omp::NVPTXGridValues;
205 if (T.isSPIRV())
206 return omp::SPIRVGridValues;
207 llvm_unreachable("No grid value available for this architecture!");
208}
209
210/// Determine which scheduling algorithm to use, determined from schedule clause
211/// arguments.
212static OMPScheduleType
213getOpenMPBaseScheduleType(llvm::omp::ScheduleKind ClauseKind, bool HasChunks,
214 bool HasSimdModifier, bool HasDistScheduleChunks) {
215 // Currently, the default schedule it static.
216 switch (ClauseKind) {
217 case OMP_SCHEDULE_Default:
218 case OMP_SCHEDULE_Static:
219 return HasChunks ? OMPScheduleType::BaseStaticChunked
220 : OMPScheduleType::BaseStatic;
221 case OMP_SCHEDULE_Dynamic:
222 return OMPScheduleType::BaseDynamicChunked;
223 case OMP_SCHEDULE_Guided:
224 return HasSimdModifier ? OMPScheduleType::BaseGuidedSimd
225 : OMPScheduleType::BaseGuidedChunked;
226 case OMP_SCHEDULE_Auto:
227 return llvm::omp::OMPScheduleType::BaseAuto;
228 case OMP_SCHEDULE_Runtime:
229 return HasSimdModifier ? OMPScheduleType::BaseRuntimeSimd
230 : OMPScheduleType::BaseRuntime;
231 case OMP_SCHEDULE_Distribute:
232 return HasDistScheduleChunks ? OMPScheduleType::BaseDistributeChunked
233 : OMPScheduleType::BaseDistribute;
234 }
235 llvm_unreachable("unhandled schedule clause argument");
236}
237
238/// Adds ordering modifier flags to schedule type.
239static OMPScheduleType
240getOpenMPOrderingScheduleType(OMPScheduleType BaseScheduleType,
241 bool HasOrderedClause) {
242 assert((BaseScheduleType & OMPScheduleType::ModifierMask) ==
243 OMPScheduleType::None &&
244 "Must not have ordering nor monotonicity flags already set");
245
246 OMPScheduleType OrderingModifier = HasOrderedClause
247 ? OMPScheduleType::ModifierOrdered
248 : OMPScheduleType::ModifierUnordered;
249 OMPScheduleType OrderingScheduleType = BaseScheduleType | OrderingModifier;
250
251 // Unsupported combinations
252 if (OrderingScheduleType ==
253 (OMPScheduleType::BaseGuidedSimd | OMPScheduleType::ModifierOrdered))
254 return OMPScheduleType::OrderedGuidedChunked;
255 else if (OrderingScheduleType == (OMPScheduleType::BaseRuntimeSimd |
256 OMPScheduleType::ModifierOrdered))
257 return OMPScheduleType::OrderedRuntime;
258
259 return OrderingScheduleType;
260}
261
262/// Adds monotonicity modifier flags to schedule type.
263static OMPScheduleType
264getOpenMPMonotonicityScheduleType(OMPScheduleType ScheduleType,
265 bool HasSimdModifier, bool HasMonotonic,
266 bool HasNonmonotonic, bool HasOrderedClause) {
267 assert((ScheduleType & OMPScheduleType::MonotonicityMask) ==
268 OMPScheduleType::None &&
269 "Must not have monotonicity flags already set");
270 assert((!HasMonotonic || !HasNonmonotonic) &&
271 "Monotonic and Nonmonotonic are contradicting each other");
272
273 if (HasMonotonic) {
274 return ScheduleType | OMPScheduleType::ModifierMonotonic;
275 } else if (HasNonmonotonic) {
276 return ScheduleType | OMPScheduleType::ModifierNonmonotonic;
277 } else {
278 // OpenMP 5.1, 2.11.4 Worksharing-Loop Construct, Description.
279 // If the static schedule kind is specified or if the ordered clause is
280 // specified, and if the nonmonotonic modifier is not specified, the
281 // effect is as if the monotonic modifier is specified. Otherwise, unless
282 // the monotonic modifier is specified, the effect is as if the
283 // nonmonotonic modifier is specified.
284 OMPScheduleType BaseScheduleType =
285 ScheduleType & ~OMPScheduleType::ModifierMask;
286 if ((BaseScheduleType == OMPScheduleType::BaseStatic) ||
287 (BaseScheduleType == OMPScheduleType::BaseStaticChunked) ||
288 HasOrderedClause) {
289 // The monotonic is used by default in openmp runtime library, so no need
290 // to set it.
291 return ScheduleType;
292 } else {
293 return ScheduleType | OMPScheduleType::ModifierNonmonotonic;
294 }
295 }
296}
297
298/// Determine the schedule type using schedule and ordering clause arguments.
299static OMPScheduleType
300computeOpenMPScheduleType(ScheduleKind ClauseKind, bool HasChunks,
301 bool HasSimdModifier, bool HasMonotonicModifier,
302 bool HasNonmonotonicModifier, bool HasOrderedClause,
303 bool HasDistScheduleChunks) {
304 OMPScheduleType BaseSchedule = getOpenMPBaseScheduleType(
305 ClauseKind, HasChunks, HasSimdModifier, HasDistScheduleChunks);
306 OMPScheduleType OrderedSchedule =
307 getOpenMPOrderingScheduleType(BaseScheduleType: BaseSchedule, HasOrderedClause);
308 OMPScheduleType Result = getOpenMPMonotonicityScheduleType(
309 ScheduleType: OrderedSchedule, HasSimdModifier, HasMonotonic: HasMonotonicModifier,
310 HasNonmonotonic: HasNonmonotonicModifier, HasOrderedClause);
311
312 assert(isValidWorkshareLoopScheduleType(Result));
313 return Result;
314}
315
316/// Given a function, if it represents the entry point of a target kernel, this
317/// returns the execution mode flags associated with that kernel.
318static std::optional<omp::OMPTgtExecModeFlags>
319getTargetKernelExecMode(Function &Kernel) {
320 CallInst *TargetInitCall = nullptr;
321 for (Instruction &Inst : Kernel.getEntryBlock()) {
322 if (auto *Call = dyn_cast<CallInst>(Val: &Inst)) {
323 if (Call->getCalledFunction()->getName() == "__kmpc_target_init") {
324 TargetInitCall = Call;
325 break;
326 }
327 }
328 }
329
330 if (!TargetInitCall)
331 return std::nullopt;
332
333 // Get the kernel mode information from the global variable associated to the
334 // first argument to the call to __kmpc_target_init. Refer to
335 // createTargetInit() to see how this is initialized.
336 Value *InitOperand = TargetInitCall->getArgOperand(i: 0);
337 GlobalVariable *KernelEnv = nullptr;
338 if (auto *Cast = dyn_cast<ConstantExpr>(Val: InitOperand))
339 KernelEnv = cast<GlobalVariable>(Val: Cast->getOperand(i_nocapture: 0));
340 else
341 KernelEnv = cast<GlobalVariable>(Val: InitOperand);
342 auto *KernelEnvInit = cast<ConstantStruct>(Val: KernelEnv->getInitializer());
343 auto *ConfigEnv = cast<ConstantStruct>(Val: KernelEnvInit->getOperand(i_nocapture: 0));
344 auto *KernelMode = cast<ConstantInt>(Val: ConfigEnv->getOperand(i_nocapture: 2));
345 return static_cast<OMPTgtExecModeFlags>(KernelMode->getZExtValue());
346}
347
348static bool isGenericKernel(Function &Fn) {
349 std::optional<omp::OMPTgtExecModeFlags> ExecMode =
350 getTargetKernelExecMode(Kernel&: Fn);
351 return !ExecMode || (*ExecMode & OMP_TGT_EXEC_MODE_GENERIC);
352}
353
354/// Make \p Source branch to \p Target.
355///
356/// Handles two situations:
357/// * \p Source already has an unconditional branch.
358/// * \p Source is a degenerate block (no terminator because the BB is
359/// the current head of the IR construction).
360static void redirectTo(BasicBlock *Source, BasicBlock *Target, DebugLoc DL) {
361 if (Instruction *Term = Source->getTerminatorOrNull()) {
362 auto *Br = cast<UncondBrInst>(Val: Term);
363 BasicBlock *Succ = Br->getSuccessor();
364 Succ->removePredecessor(Pred: Source, /*KeepOneInputPHIs=*/true);
365 Br->setSuccessor(Target);
366 return;
367 }
368
369 auto *NewBr = UncondBrInst::Create(Target, InsertBefore: Source);
370 NewBr->setDebugLoc(DL);
371}
372
373void llvm::spliceBB(IRBuilderBase::InsertPoint IP, BasicBlock *New,
374 bool CreateBranch, DebugLoc DL) {
375 assert(New->getFirstInsertionPt() == New->begin() &&
376 "Target BB must not have PHI nodes");
377
378 // Move instructions to new block.
379 BasicBlock *Old = IP.getBlock();
380 // If the `Old` block is empty then there are no instructions to move. But in
381 // the new debug scheme, it could have trailing debug records which will be
382 // moved to `New` in `spliceDebugInfoEmptyBlock`. We dont want that for 2
383 // reasons:
384 // 1. If `New` is also empty, `BasicBlock::splice` crashes.
385 // 2. Even if `New` is not empty, the rationale to move those records to `New`
386 // (in `spliceDebugInfoEmptyBlock`) does not apply here. That function
387 // assumes that `Old` is optimized out and is going away. This is not the case
388 // here. The `Old` block is still being used e.g. a branch instruction is
389 // added to it later in this function.
390 // So we call `BasicBlock::splice` only when `Old` is not empty.
391 if (!Old->empty())
392 New->splice(ToIt: New->begin(), FromBB: Old, FromBeginIt: IP.getPoint(), FromEndIt: Old->end());
393
394 if (CreateBranch) {
395 auto *NewBr = UncondBrInst::Create(Target: New, InsertBefore: Old);
396 NewBr->setDebugLoc(DL);
397 }
398}
399
400void llvm::spliceBB(IRBuilder<> &Builder, BasicBlock *New, bool CreateBranch) {
401 DebugLoc DebugLoc = Builder.getCurrentDebugLocation();
402 BasicBlock *Old = Builder.GetInsertBlock();
403
404 spliceBB(IP: Builder.saveIP(), New, CreateBranch, DL: DebugLoc);
405 if (CreateBranch)
406 Builder.SetInsertPoint(Old->getTerminator());
407 else
408 Builder.SetInsertPoint(Old);
409
410 // SetInsertPoint also updates the Builder's debug location, but we want to
411 // keep the one the Builder was configured to use.
412 Builder.SetCurrentDebugLocation(DebugLoc);
413}
414
415BasicBlock *llvm::splitBB(IRBuilderBase::InsertPoint IP, bool CreateBranch,
416 DebugLoc DL, llvm::Twine Name) {
417 BasicBlock *Old = IP.getBlock();
418 BasicBlock *New = BasicBlock::Create(
419 Context&: Old->getContext(), Name: Name.isTriviallyEmpty() ? Old->getName() : Name,
420 Parent: Old->getParent(), InsertBefore: Old->getNextNode());
421 spliceBB(IP, New, CreateBranch, DL);
422 New->replaceSuccessorsPhiUsesWith(Old, New);
423 return New;
424}
425
426BasicBlock *llvm::splitBB(IRBuilderBase &Builder, bool CreateBranch,
427 llvm::Twine Name) {
428 DebugLoc DebugLoc = Builder.getCurrentDebugLocation();
429 BasicBlock *New = splitBB(IP: Builder.saveIP(), CreateBranch, DL: DebugLoc, Name);
430 if (CreateBranch)
431 Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator());
432 else
433 Builder.SetInsertPoint(Builder.GetInsertBlock());
434 // SetInsertPoint also updates the Builder's debug location, but we want to
435 // keep the one the Builder was configured to use.
436 Builder.SetCurrentDebugLocation(DebugLoc);
437 return New;
438}
439
440BasicBlock *llvm::splitBB(IRBuilder<> &Builder, bool CreateBranch,
441 llvm::Twine Name) {
442 DebugLoc DebugLoc = Builder.getCurrentDebugLocation();
443 BasicBlock *New = splitBB(IP: Builder.saveIP(), CreateBranch, DL: DebugLoc, Name);
444 if (CreateBranch)
445 Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator());
446 else
447 Builder.SetInsertPoint(Builder.GetInsertBlock());
448 // SetInsertPoint also updates the Builder's debug location, but we want to
449 // keep the one the Builder was configured to use.
450 Builder.SetCurrentDebugLocation(DebugLoc);
451 return New;
452}
453
454BasicBlock *llvm::splitBBWithSuffix(IRBuilderBase &Builder, bool CreateBranch,
455 llvm::Twine Suffix) {
456 BasicBlock *Old = Builder.GetInsertBlock();
457 return splitBB(Builder, CreateBranch, Name: Old->getName() + Suffix);
458}
459
460// This function creates a fake integer value and a fake use for the integer
461// value. It returns the fake value created. This is useful in modeling the
462// extra arguments to the outlined functions.
463Value *createFakeIntVal(IRBuilderBase &Builder,
464 OpenMPIRBuilder::InsertPointTy OuterAllocaIP,
465 llvm::SmallVectorImpl<Instruction *> &ToBeDeleted,
466 OpenMPIRBuilder::InsertPointTy InnerAllocaIP,
467 const Twine &Name = "", bool AsPtr = true,
468 bool Is64Bit = false) {
469 Builder.restoreIP(IP: OuterAllocaIP);
470 IntegerType *IntTy = Is64Bit ? Builder.getInt64Ty() : Builder.getInt32Ty();
471 Instruction *FakeVal;
472 AllocaInst *FakeValAddr =
473 Builder.CreateAlloca(Ty: IntTy, ArraySize: nullptr, Name: Name + ".addr");
474 ToBeDeleted.push_back(Elt: FakeValAddr);
475
476 if (AsPtr) {
477 FakeVal = FakeValAddr;
478 // The runtime passes these extra arguments to the outlined function as
479 // generic pointers, so cast away a non-zero alloca address space.
480 if (FakeValAddr->getAddressSpace() != 0) {
481 FakeVal = cast<Instruction>(Val: Builder.CreateAddrSpaceCast(
482 V: FakeValAddr, DestTy: Builder.getPtrTy(), Name: Name + ".ascast"));
483 ToBeDeleted.push_back(Elt: FakeVal);
484 }
485 } else {
486 FakeVal = Builder.CreateLoad(Ty: IntTy, Ptr: FakeValAddr, Name: Name + ".val");
487 ToBeDeleted.push_back(Elt: FakeVal);
488 }
489
490 // Generate a fake use of this value
491 Builder.restoreIP(IP: InnerAllocaIP);
492 Instruction *UseFakeVal;
493 if (AsPtr) {
494 UseFakeVal = Builder.CreateLoad(Ty: IntTy, Ptr: FakeVal, Name: Name + ".use");
495 } else {
496 UseFakeVal = cast<BinaryOperator>(Val: Builder.CreateAdd(
497 LHS: FakeVal, RHS: Is64Bit ? Builder.getInt64(C: 10) : Builder.getInt32(C: 10)));
498 }
499 ToBeDeleted.push_back(Elt: UseFakeVal);
500 return FakeVal;
501}
502
503//===----------------------------------------------------------------------===//
504// OpenMPIRBuilderConfig
505//===----------------------------------------------------------------------===//
506
507namespace {
508LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();
509/// Values for bit flags for marking which requires clauses have been used.
510enum OpenMPOffloadingRequiresDirFlags {
511 /// flag undefined.
512 OMP_REQ_UNDEFINED = 0x000,
513 /// no requires directive present.
514 OMP_REQ_NONE = 0x001,
515 /// reverse_offload clause.
516 OMP_REQ_REVERSE_OFFLOAD = 0x002,
517 /// unified_address clause.
518 OMP_REQ_UNIFIED_ADDRESS = 0x004,
519 /// unified_shared_memory clause.
520 OMP_REQ_UNIFIED_SHARED_MEMORY = 0x008,
521 /// dynamic_allocators clause.
522 OMP_REQ_DYNAMIC_ALLOCATORS = 0x010,
523 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_REQ_DYNAMIC_ALLOCATORS)
524};
525
526class OMPCodeExtractor : public CodeExtractor {
527public:
528 OMPCodeExtractor(OpenMPIRBuilder &OMPBuilder, ArrayRef<BasicBlock *> BBs,
529 DominatorTree *DT = nullptr, bool AggregateArgs = false,
530 BlockFrequencyInfo *BFI = nullptr,
531 BranchProbabilityInfo *BPI = nullptr,
532 AssumptionCache *AC = nullptr, bool AllowVarArgs = false,
533 bool AllowAlloca = false,
534 BasicBlock *AllocationBlock = nullptr,
535 ArrayRef<BasicBlock *> DeallocationBlocks = {},
536 std::string Suffix = "", bool ArgsInZeroAddressSpace = false)
537 : CodeExtractor(BBs, DT, AggregateArgs, BFI, BPI, AC, AllowVarArgs,
538 AllowAlloca, AllocationBlock, DeallocationBlocks, Suffix,
539 ArgsInZeroAddressSpace),
540 OMPBuilder(OMPBuilder) {}
541
542 virtual ~OMPCodeExtractor() = default;
543
544protected:
545 OpenMPIRBuilder &OMPBuilder;
546};
547
548class DeviceSharedMemCodeExtractor : public OMPCodeExtractor {
549public:
550 using OMPCodeExtractor::OMPCodeExtractor;
551 virtual ~DeviceSharedMemCodeExtractor() = default;
552
553protected:
554 virtual Instruction *
555 allocateVar(IRBuilder<>::InsertPoint AllocaIP, DebugLoc DL, Type *VarType,
556 const Twine &Name = Twine(""),
557 AddrSpaceCastInst **CastedAlloc = nullptr) override {
558 return OMPBuilder.createOMPAllocShared(Loc: {AllocaIP, DL}, VarType, Name);
559 }
560
561 virtual Instruction *deallocateVar(IRBuilder<>::InsertPoint DeallocIP,
562 DebugLoc DL, Value *Var,
563 Type *VarType) override {
564 return OMPBuilder.createOMPFreeShared(Loc: {DeallocIP, DL}, Addr: Var, VarType);
565 }
566};
567
568/// Helper storing information about regions to outline using device shared
569/// memory for intermediate allocations.
570struct DeviceSharedMemOutlineInfo : public OpenMPIRBuilder::OutlineInfo {
571 OpenMPIRBuilder &OMPBuilder;
572
573 DeviceSharedMemOutlineInfo(OpenMPIRBuilder &OMPBuilder)
574 : OMPBuilder(OMPBuilder) {}
575 virtual ~DeviceSharedMemOutlineInfo() = default;
576
577 virtual std::unique_ptr<CodeExtractor>
578 createCodeExtractor(ArrayRef<BasicBlock *> Blocks,
579 bool ArgsInZeroAddressSpace,
580 Twine Suffix = Twine("")) override;
581};
582
583} // anonymous namespace
584
585OpenMPIRBuilderConfig::OpenMPIRBuilderConfig()
586 : RequiresFlags(OMP_REQ_UNDEFINED) {}
587
588OpenMPIRBuilderConfig::OpenMPIRBuilderConfig(
589 bool IsTargetDevice, bool IsGPU, bool OpenMPOffloadMandatory,
590 bool HasRequiresReverseOffload, bool HasRequiresUnifiedAddress,
591 bool HasRequiresUnifiedSharedMemory, bool HasRequiresDynamicAllocators)
592 : IsTargetDevice(IsTargetDevice), IsGPU(IsGPU),
593 OpenMPOffloadMandatory(OpenMPOffloadMandatory),
594 RequiresFlags(OMP_REQ_UNDEFINED) {
595 if (HasRequiresReverseOffload)
596 RequiresFlags |= OMP_REQ_REVERSE_OFFLOAD;
597 if (HasRequiresUnifiedAddress)
598 RequiresFlags |= OMP_REQ_UNIFIED_ADDRESS;
599 if (HasRequiresUnifiedSharedMemory)
600 RequiresFlags |= OMP_REQ_UNIFIED_SHARED_MEMORY;
601 if (HasRequiresDynamicAllocators)
602 RequiresFlags |= OMP_REQ_DYNAMIC_ALLOCATORS;
603}
604
605bool OpenMPIRBuilderConfig::hasRequiresReverseOffload() const {
606 return RequiresFlags & OMP_REQ_REVERSE_OFFLOAD;
607}
608
609bool OpenMPIRBuilderConfig::hasRequiresUnifiedAddress() const {
610 return RequiresFlags & OMP_REQ_UNIFIED_ADDRESS;
611}
612
613bool OpenMPIRBuilderConfig::hasRequiresUnifiedSharedMemory() const {
614 return RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY;
615}
616
617bool OpenMPIRBuilderConfig::hasRequiresDynamicAllocators() const {
618 return RequiresFlags & OMP_REQ_DYNAMIC_ALLOCATORS;
619}
620
621int64_t OpenMPIRBuilderConfig::getRequiresFlags() const {
622 return hasRequiresFlags() ? RequiresFlags
623 : static_cast<int64_t>(OMP_REQ_NONE);
624}
625
626void OpenMPIRBuilderConfig::setHasRequiresReverseOffload(bool Value) {
627 if (Value)
628 RequiresFlags |= OMP_REQ_REVERSE_OFFLOAD;
629 else
630 RequiresFlags &= ~OMP_REQ_REVERSE_OFFLOAD;
631}
632
633void OpenMPIRBuilderConfig::setHasRequiresUnifiedAddress(bool Value) {
634 if (Value)
635 RequiresFlags |= OMP_REQ_UNIFIED_ADDRESS;
636 else
637 RequiresFlags &= ~OMP_REQ_UNIFIED_ADDRESS;
638}
639
640void OpenMPIRBuilderConfig::setHasRequiresUnifiedSharedMemory(bool Value) {
641 if (Value)
642 RequiresFlags |= OMP_REQ_UNIFIED_SHARED_MEMORY;
643 else
644 RequiresFlags &= ~OMP_REQ_UNIFIED_SHARED_MEMORY;
645}
646
647void OpenMPIRBuilderConfig::setHasRequiresDynamicAllocators(bool Value) {
648 if (Value)
649 RequiresFlags |= OMP_REQ_DYNAMIC_ALLOCATORS;
650 else
651 RequiresFlags &= ~OMP_REQ_DYNAMIC_ALLOCATORS;
652}
653
654//===----------------------------------------------------------------------===//
655// OpenMPIRBuilder
656//===----------------------------------------------------------------------===//
657
658void OpenMPIRBuilder::getKernelArgsVector(TargetKernelArgs &KernelArgs,
659 IRBuilderBase &Builder,
660 SmallVector<Value *> &ArgsVector) {
661 Value *Version = Builder.getInt32(OMP_KERNEL_ARG_VERSION);
662 Value *PointerNum = Builder.getInt32(C: KernelArgs.NumTargetItems);
663 auto Int32Ty = Type::getInt32Ty(C&: Builder.getContext());
664 constexpr size_t MaxDim = 3;
665 Value *ZeroArray = Constant::getNullValue(Ty: ArrayType::get(ElementType: Int32Ty, NumElements: MaxDim));
666
667 Value *HasNoWaitFlag = Builder.getInt64(C: KernelArgs.HasNoWait);
668
669 Value *DynCGroupMemFallbackFlag =
670 Builder.getInt64(C: static_cast<uint64_t>(KernelArgs.DynCGroupMemFallback));
671 DynCGroupMemFallbackFlag = Builder.CreateShl(LHS: DynCGroupMemFallbackFlag, RHS: 2);
672
673 Value *StrictBlocksFlag = Builder.getInt64(C: KernelArgs.StrictBlocks);
674 Value *StrictThreadsFlag = Builder.getInt64(C: KernelArgs.StrictThreads);
675
676 StrictBlocksFlag = Builder.CreateShl(LHS: StrictBlocksFlag, RHS: 6);
677 StrictThreadsFlag = Builder.CreateShl(LHS: StrictThreadsFlag, RHS: 7);
678
679 Value *Flags = Builder.CreateOr(LHS: HasNoWaitFlag, RHS: DynCGroupMemFallbackFlag);
680 Flags = Builder.CreateOr(LHS: Flags, RHS: StrictBlocksFlag);
681 Flags = Builder.CreateOr(LHS: Flags, RHS: StrictThreadsFlag);
682
683 assert(!KernelArgs.NumTeams.empty() && !KernelArgs.NumThreads.empty());
684
685 Value *NumTeams3D =
686 Builder.CreateInsertValue(Agg: ZeroArray, Val: KernelArgs.NumTeams[0], Idxs: {0});
687 Value *NumThreads3D =
688 Builder.CreateInsertValue(Agg: ZeroArray, Val: KernelArgs.NumThreads[0], Idxs: {0});
689 for (unsigned I :
690 seq<unsigned>(Begin: 1, End: std::min(a: KernelArgs.NumTeams.size(), b: MaxDim)))
691 NumTeams3D =
692 Builder.CreateInsertValue(Agg: NumTeams3D, Val: KernelArgs.NumTeams[I], Idxs: {I});
693 for (unsigned I :
694 seq<unsigned>(Begin: 1, End: std::min(a: KernelArgs.NumThreads.size(), b: MaxDim)))
695 NumThreads3D =
696 Builder.CreateInsertValue(Agg: NumThreads3D, Val: KernelArgs.NumThreads[I], Idxs: {I});
697
698 ArgsVector = {Version,
699 PointerNum,
700 KernelArgs.RTArgs.BasePointersArray,
701 KernelArgs.RTArgs.PointersArray,
702 KernelArgs.RTArgs.SizesArray,
703 KernelArgs.RTArgs.MapTypesArray,
704 KernelArgs.RTArgs.MapNamesArray,
705 KernelArgs.RTArgs.MappersArray,
706 KernelArgs.NumIterations,
707 Flags,
708 NumTeams3D,
709 NumThreads3D,
710 KernelArgs.DynCGroupMem};
711}
712
713void OpenMPIRBuilder::addAttributes(omp::RuntimeFunction FnID, Function &Fn) {
714 LLVMContext &Ctx = Fn.getContext();
715
716 // Get the function's current attributes.
717 auto Attrs = Fn.getAttributes();
718 auto FnAttrs = Attrs.getFnAttrs();
719 auto RetAttrs = Attrs.getRetAttrs();
720 SmallVector<AttributeSet, 4> ArgAttrs;
721 for (size_t ArgNo = 0; ArgNo < Fn.arg_size(); ++ArgNo)
722 ArgAttrs.emplace_back(Args: Attrs.getParamAttrs(ArgNo));
723
724 // Add AS to FnAS while taking special care with integer extensions.
725 auto addAttrSet = [&](AttributeSet &FnAS, const AttributeSet &AS,
726 bool Param = true) -> void {
727 bool HasSignExt = AS.hasAttribute(Kind: Attribute::SExt);
728 bool HasZeroExt = AS.hasAttribute(Kind: Attribute::ZExt);
729 if (HasSignExt || HasZeroExt) {
730 assert(AS.getNumAttributes() == 1 &&
731 "Currently not handling extension attr combined with others.");
732 if (Param) {
733 if (auto AK = TargetLibraryInfo::getExtAttrForI32Param(T, Signed: HasSignExt))
734 FnAS = FnAS.addAttribute(C&: Ctx, Kind: AK);
735 } else if (auto AK =
736 TargetLibraryInfo::getExtAttrForI32Return(T, Signed: HasSignExt))
737 FnAS = FnAS.addAttribute(C&: Ctx, Kind: AK);
738 } else {
739 FnAS = FnAS.addAttributes(C&: Ctx, AS);
740 }
741 };
742
743#define OMP_ATTRS_SET(VarName, AttrSet) AttributeSet VarName = AttrSet;
744#include "llvm/Frontend/OpenMP/OMPKinds.def"
745
746 // Add attributes to the function declaration.
747 switch (FnID) {
748#define OMP_RTL_ATTRS(Enum, FnAttrSet, RetAttrSet, ArgAttrSets) \
749 case Enum: \
750 FnAttrs = FnAttrs.addAttributes(Ctx, FnAttrSet); \
751 addAttrSet(RetAttrs, RetAttrSet, /*Param*/ false); \
752 for (size_t ArgNo = 0; ArgNo < ArgAttrSets.size(); ++ArgNo) \
753 addAttrSet(ArgAttrs[ArgNo], ArgAttrSets[ArgNo]); \
754 Fn.setAttributes(AttributeList::get(Ctx, FnAttrs, RetAttrs, ArgAttrs)); \
755 break;
756#include "llvm/Frontend/OpenMP/OMPKinds.def"
757 default:
758 // Attributes are optional.
759 break;
760 }
761}
762
763FunctionCallee
764OpenMPIRBuilder::getOrCreateRuntimeFunction(Module &M, RuntimeFunction FnID) {
765 FunctionType *FnTy = nullptr;
766 Function *Fn = nullptr;
767
768 // Try to find the declation in the module first.
769 switch (FnID) {
770#define OMP_RTL(Enum, Str, IsVarArg, ReturnType, ...) \
771 case Enum: \
772 FnTy = FunctionType::get(ReturnType, ArrayRef<Type *>{__VA_ARGS__}, \
773 IsVarArg); \
774 Fn = M.getFunction(Str); \
775 break;
776#include "llvm/Frontend/OpenMP/OMPKinds.def"
777 }
778
779 if (!Fn) {
780 // Create a new declaration if we need one.
781 switch (FnID) {
782#define OMP_RTL(Enum, Str, ...) \
783 case Enum: \
784 Fn = Function::Create(FnTy, GlobalValue::ExternalLinkage, Str, M); \
785 break;
786#include "llvm/Frontend/OpenMP/OMPKinds.def"
787 }
788 Fn->setCallingConv(Config.getRuntimeCC());
789 // Add information if the runtime function takes a callback function
790 if (FnID == OMPRTL___kmpc_fork_call || FnID == OMPRTL___kmpc_fork_teams) {
791 if (!Fn->hasMetadata(KindID: LLVMContext::MD_callback)) {
792 LLVMContext &Ctx = Fn->getContext();
793 MDBuilder MDB(Ctx);
794 // Annotate the callback behavior of the runtime function:
795 // - The callback callee is argument number 2 (microtask).
796 // - The first two arguments of the callback callee are unknown (-1).
797 // - All variadic arguments to the runtime function are passed to the
798 // callback callee.
799 Fn->addMetadata(
800 KindID: LLVMContext::MD_callback,
801 MD&: *MDNode::get(Context&: Ctx, MDs: {MDB.createCallbackEncoding(
802 CalleeArgNo: 2, Arguments: {-1, -1}, /* VarArgsArePassed */ true)}));
803 }
804 }
805
806 LLVM_DEBUG(dbgs() << "Created OpenMP runtime function " << Fn->getName()
807 << " with type " << *Fn->getFunctionType() << "\n");
808 addAttributes(FnID, Fn&: *Fn);
809
810 } else {
811 LLVM_DEBUG(dbgs() << "Found OpenMP runtime function " << Fn->getName()
812 << " with type " << *Fn->getFunctionType() << "\n");
813 }
814
815 assert(Fn && "Failed to create OpenMP runtime function");
816
817 return {FnTy, Fn};
818}
819
820Expected<BasicBlock *>
821OpenMPIRBuilder::FinalizationInfo::getFiniBB(IRBuilderBase &Builder) {
822 if (!FiniBB) {
823 Function *ParentFunc = Builder.GetInsertBlock()->getParent();
824 IRBuilderBase::InsertPointGuard Guard(Builder);
825 FiniBB = BasicBlock::Create(Context&: Builder.getContext(), Name: ".fini", Parent: ParentFunc);
826 Builder.SetInsertPoint(FiniBB);
827 // FiniCB adds the branch to the exit stub.
828 if (Error Err = FiniCB(Builder.saveIP()))
829 return Err;
830 }
831 return FiniBB;
832}
833
834Error OpenMPIRBuilder::FinalizationInfo::mergeFiniBB(IRBuilderBase &Builder,
835 BasicBlock *OtherFiniBB) {
836 // Simple case: FiniBB does not exist yet: re-use OtherFiniBB.
837 if (!FiniBB) {
838 FiniBB = OtherFiniBB;
839
840 Builder.SetInsertPoint(FiniBB->getFirstNonPHIIt());
841 if (Error Err = FiniCB(Builder.saveIP()))
842 return Err;
843
844 return Error::success();
845 }
846
847 // Move instructions from FiniBB to the start of OtherFiniBB.
848 auto EndIt = FiniBB->end();
849 if (FiniBB->size() >= 1)
850 if (auto Prev = std::prev(x: EndIt); Prev->isTerminator())
851 EndIt = Prev;
852 OtherFiniBB->splice(ToIt: OtherFiniBB->getFirstNonPHIIt(), FromBB: FiniBB, FromBeginIt: FiniBB->begin(),
853 FromEndIt: EndIt);
854
855 FiniBB->replaceAllUsesWith(V: OtherFiniBB);
856 FiniBB->eraseFromParent();
857 FiniBB = OtherFiniBB;
858 return Error::success();
859}
860
861Function *OpenMPIRBuilder::getOrCreateRuntimeFunctionPtr(RuntimeFunction FnID) {
862 FunctionCallee RTLFn = getOrCreateRuntimeFunction(M, FnID);
863 auto *Fn = dyn_cast<llvm::Function>(Val: RTLFn.getCallee());
864 assert(Fn && "Failed to create OpenMP runtime function pointer");
865 return Fn;
866}
867
868CallInst *OpenMPIRBuilder::createRuntimeFunctionCall(FunctionCallee Callee,
869 ArrayRef<Value *> Args,
870 StringRef Name) {
871 CallInst *Call = Builder.CreateCall(Callee, Args, Name);
872 Call->setCallingConv(Config.getRuntimeCC());
873 return Call;
874}
875
876void OpenMPIRBuilder::initialize() { initializeTypes(M); }
877
878static void raiseUserConstantDataAllocasToEntryBlock(IRBuilderBase &Builder,
879 Function *Function) {
880 BasicBlock &EntryBlock = Function->getEntryBlock();
881 BasicBlock::iterator MoveLocInst = EntryBlock.getFirstNonPHIIt();
882
883 // Loop over blocks looking for constant allocas, skipping the entry block
884 // as any allocas there are already in the desired location.
885 for (auto Block = std::next(x: Function->begin(), n: 1); Block != Function->end();
886 Block++) {
887 for (auto Inst = Block->getReverseIterator()->begin();
888 Inst != Block->getReverseIterator()->end();) {
889 if (auto *AllocaInst = dyn_cast_if_present<llvm::AllocaInst>(Val&: Inst)) {
890 Inst++;
891 if (!isa<ConstantData>(Val: AllocaInst->getArraySize()))
892 continue;
893 AllocaInst->moveBeforePreserving(MovePos: MoveLocInst);
894 } else {
895 Inst++;
896 }
897 }
898 }
899}
900
901static void hoistNonEntryAllocasToEntryBlock(llvm::BasicBlock &Block) {
902 llvm::SmallVector<llvm::Instruction *> AllocasToMove;
903
904 auto ShouldHoistAlloca = [](const llvm::AllocaInst &AllocaInst) {
905 // TODO: For now, we support simple static allocations, we might need to
906 // move non-static ones as well. However, this will need further analysis to
907 // move the lenght arguments as well.
908 return !AllocaInst.isArrayAllocation();
909 };
910
911 for (llvm::Instruction &Inst : Block)
912 if (auto *AllocaInst = llvm::dyn_cast<llvm::AllocaInst>(Val: &Inst))
913 if (ShouldHoistAlloca(*AllocaInst))
914 AllocasToMove.push_back(Elt: AllocaInst);
915
916 auto InsertPoint =
917 Block.getParent()->getEntryBlock().getTerminator()->getIterator();
918
919 for (llvm::Instruction *AllocaInst : AllocasToMove)
920 AllocaInst->moveBefore(InsertPos: InsertPoint);
921}
922
923static void hoistNonEntryAllocasToEntryBlock(llvm::Function *Func) {
924 PostDominatorTree PostDomTree(*Func);
925 for (llvm::BasicBlock &BB : *Func)
926 if (PostDomTree.properlyDominates(A: &BB, B: &Func->getEntryBlock()))
927 hoistNonEntryAllocasToEntryBlock(Block&: BB);
928}
929
930void OpenMPIRBuilder::finalize(Function *Fn) {
931 SmallPtrSet<BasicBlock *, 32> ParallelRegionBlockSet;
932 SmallVector<BasicBlock *, 32> Blocks;
933 SmallVector<std::unique_ptr<OutlineInfo>, 16> DeferredOutlines;
934 for (std::unique_ptr<OutlineInfo> &OI : OutlineInfos) {
935 // Skip functions that have not finalized yet; may happen with nested
936 // function generation.
937 if (Fn && OI->getFunction() != Fn) {
938 DeferredOutlines.push_back(Elt: std::move(OI));
939 continue;
940 }
941
942 ParallelRegionBlockSet.clear();
943 Blocks.clear();
944 OI->collectBlocks(BlockSet&: ParallelRegionBlockSet, BlockVector&: Blocks);
945
946 Function *OuterFn = OI->getFunction();
947 CodeExtractorAnalysisCache CEAC(*OuterFn);
948 // If we generate code for the target device, we need to allocate
949 // struct for aggregate params in the device default alloca address space.
950 // OpenMP runtime requires that the params of the extracted functions are
951 // passed as zero address space pointers. This flag ensures that
952 // CodeExtractor generates correct code for extracted functions
953 // which are used by OpenMP runtime.
954 bool ArgsInZeroAddressSpace = Config.isTargetDevice();
955 std::unique_ptr<CodeExtractor> Extractor =
956 OI->createCodeExtractor(Blocks, ArgsInZeroAddressSpace, Suffix: ".omp_par");
957
958 LLVM_DEBUG(dbgs() << "Before outlining: " << *OuterFn << "\n");
959 LLVM_DEBUG(dbgs() << "Entry " << OI->EntryBB->getName()
960 << " Exit: " << OI->ExitBB->getName() << "\n");
961 assert(Extractor->isEligible() &&
962 "Expected OpenMP outlining to be possible!");
963
964 for (auto *V : OI->ExcludeArgsFromAggregate)
965 Extractor->excludeArgFromAggregate(Arg: V);
966
967 Function *OutlinedFn =
968 Extractor->extractCodeRegion(CEAC, Inputs&: OI->Inputs, Outputs&: OI->Outputs);
969
970 // Forward target-cpu, target-features attributes to the outlined function.
971 auto TargetCpuAttr = OuterFn->getFnAttribute(Kind: "target-cpu");
972 if (TargetCpuAttr.isStringAttribute())
973 OutlinedFn->addFnAttr(Attr: TargetCpuAttr);
974
975 auto TargetFeaturesAttr = OuterFn->getFnAttribute(Kind: "target-features");
976 if (TargetFeaturesAttr.isStringAttribute())
977 OutlinedFn->addFnAttr(Attr: TargetFeaturesAttr);
978
979 LLVM_DEBUG(dbgs() << "After outlining: " << *OuterFn << "\n");
980 LLVM_DEBUG(dbgs() << " Outlined function: " << *OutlinedFn << "\n");
981 assert(OutlinedFn->getReturnType()->isVoidTy() &&
982 "OpenMP outlined functions should not return a value!");
983
984 // For compability with the clang CG we move the outlined function after the
985 // one with the parallel region.
986 OutlinedFn->removeFromParent();
987 M.getFunctionList().insertAfter(where: OuterFn->getIterator(), New: OutlinedFn);
988
989 // Remove the artificial entry introduced by the extractor right away, we
990 // made our own entry block after all.
991 {
992 BasicBlock &ArtificialEntry = OutlinedFn->getEntryBlock();
993 assert(ArtificialEntry.getUniqueSuccessor() == OI->EntryBB);
994 assert(OI->EntryBB->getUniquePredecessor() == &ArtificialEntry);
995 // Move instructions from the to-be-deleted ArtificialEntry to the entry
996 // basic block of the parallel region. CodeExtractor generates
997 // instructions to unwrap the aggregate argument and may sink
998 // allocas/bitcasts for values that are solely used in the outlined region
999 // and do not escape.
1000 assert(!ArtificialEntry.empty() &&
1001 "Expected instructions to add in the outlined region entry");
1002 for (BasicBlock::reverse_iterator It = ArtificialEntry.rbegin(),
1003 End = ArtificialEntry.rend();
1004 It != End;) {
1005 Instruction &I = *It;
1006 It++;
1007
1008 if (I.isTerminator()) {
1009 // Absorb any debug value that terminator may have
1010 if (Instruction *TI = OI->EntryBB->getTerminatorOrNull())
1011 TI->adoptDbgRecords(BB: &ArtificialEntry, It: I.getIterator(), InsertAtHead: false);
1012 continue;
1013 }
1014
1015 I.moveBeforePreserving(BB&: *OI->EntryBB,
1016 I: OI->EntryBB->getFirstInsertionPt());
1017 }
1018
1019 OI->EntryBB->moveBefore(MovePos: &ArtificialEntry);
1020 ArtificialEntry.eraseFromParent();
1021 }
1022 assert(&OutlinedFn->getEntryBlock() == OI->EntryBB);
1023 assert(OutlinedFn && OutlinedFn->hasNUses(1));
1024
1025 // Run a user callback, e.g. to add attributes.
1026 if (OI->PostOutlineCB)
1027 OI->PostOutlineCB(*OutlinedFn);
1028
1029 if (OI->FixUpNonEntryAllocas)
1030 hoistNonEntryAllocasToEntryBlock(Func: OutlinedFn);
1031 }
1032
1033 // Remove work items that have been completed.
1034 OutlineInfos = std::move(DeferredOutlines);
1035
1036 // The createTarget functions embeds user written code into
1037 // the target region which may inject allocas which need to
1038 // be moved to the entry block of our target or risk malformed
1039 // optimisations by later passes, this is only relevant for
1040 // the device pass which appears to be a little more delicate
1041 // when it comes to optimisations (however, we do not block on
1042 // that here, it's up to the inserter to the list to do so).
1043 // This notbaly has to occur after the OutlinedInfo candidates
1044 // have been extracted so we have an end product that will not
1045 // be implicitly adversely affected by any raises unless
1046 // intentionally appended to the list.
1047 // NOTE: This only does so for ConstantData, it could be extended
1048 // to ConstantExpr's with further effort, however, they should
1049 // largely be folded when they get here. Extending it to runtime
1050 // defined/read+writeable allocation sizes would be non-trivial
1051 // (need to factor in movement of any stores to variables the
1052 // allocation size depends on, as well as the usual loads,
1053 // otherwise it'll yield the wrong result after movement) and
1054 // likely be more suitable as an LLVM optimisation pass.
1055 for (Function *F : ConstantAllocaRaiseCandidates)
1056 raiseUserConstantDataAllocasToEntryBlock(Builder, Function: F);
1057
1058 EmitMetadataErrorReportFunctionTy &&ErrorReportFn =
1059 [](EmitMetadataErrorKind Kind,
1060 const TargetRegionEntryInfo &EntryInfo) -> void {
1061 errs() << "Error of kind: " << Kind
1062 << " when emitting offload entries and metadata during "
1063 "OMPIRBuilder finalization \n";
1064 };
1065
1066 if (!OffloadInfoManager.empty())
1067 createOffloadEntriesAndInfoMetadata(ErrorReportFunction&: ErrorReportFn);
1068
1069 // Rewrite uses of globals to their replacement declare target globals if
1070 // we are processing a device module.
1071 if (Config.isTargetDevice())
1072 applyDeclareTargetGlobalReplacements();
1073
1074 if (Config.EmitLLVMUsedMetaInfo.value_or(u: false)) {
1075 std::vector<WeakTrackingVH> LLVMCompilerUsed = {
1076 M.getGlobalVariable(Name: "__openmp_nvptx_data_transfer_temporary_storage")};
1077 emitUsed(Name: "llvm.compiler.used", List: LLVMCompilerUsed);
1078 }
1079
1080 IsFinalized = true;
1081}
1082
1083bool OpenMPIRBuilder::isFinalized() { return IsFinalized; }
1084
1085void OpenMPIRBuilder::registerDeclareTargetGlobalReplacement(
1086 GlobalValue *Original, GlobalValue *Replacement) {
1087 assert(Original && Replacement &&
1088 "Null values provided to registerDeclareTargetGlobalReplacement");
1089 DeclareTargetGlobalReplacements.push_back(Elt: {.Original: Original, .Replacement: Replacement});
1090}
1091
1092void OpenMPIRBuilder::applyDeclareTargetGlobalReplacements() {
1093 for (DeclareTargetGlobalReplacement &R : DeclareTargetGlobalReplacements) {
1094 GlobalValue *OldGV = R.Original;
1095 GlobalValue *NewGV = R.Replacement;
1096
1097 assert(OldGV && NewGV &&
1098 "A null value was inserted into DeclareTargetGlobalReplacements");
1099
1100 // The assert above should catch this case, but this is kept to attempt
1101 // to proceed without issue when asserts are off.
1102 if (!OldGV || !NewGV)
1103 continue;
1104
1105 // The replacement global is a reference pointer that holds the
1106 // address of the device-resident storage. Every use must load the
1107 // reference pointer first and use the loaded address.
1108 //
1109 // Constant expression users (e.g. a constant GEP embedded in another
1110 // global's initializer or in an instruction) cannot have a load inserted
1111 // in place, so first expand any constant-expression users that live inside
1112 // functions into instructions. Any remaining constant users are handled
1113 // via a direct constant rewrite below as we cannot materialize a load
1114 // there.
1115 //
1116 // NOTE: We extend the constant rewrite to module scope, as we replace all
1117 // usages.
1118 if (auto *OldConst = dyn_cast<Constant>(Val: OldGV))
1119 convertUsersOfConstantsToInstructions(Consts: OldConst,
1120 /*RestrictToFunc=*/nullptr,
1121 /*RemoveDeadConstants=*/false);
1122
1123 IRBuilderBase::InsertPointGuard Guard(Builder);
1124 SmallVector<User *, 16> Users(OldGV->users());
1125 for (User *U : Users) {
1126 auto *Insn = dyn_cast<Instruction>(Val: U);
1127 if (!Insn)
1128 continue;
1129
1130 // A PHI node cannot have a load inserted immediately before it, as PHIs
1131 // must remain grouped at the top of their basic block. So we need to
1132 // make sure any loads we emit are generated in the preceding edge, a
1133 // PHI may reference the global on more than one edge, so every matching
1134 // slot must be handled.
1135 if (auto *PHI = dyn_cast<PHINode>(Val: Insn)) {
1136 for (unsigned I = 0, E = PHI->getNumIncomingValues(); I < E; ++I) {
1137 if (PHI->getIncomingValue(i: I) != OldGV)
1138 continue;
1139
1140 BasicBlock *IncomingBB = PHI->getIncomingBlock(i: I);
1141 Builder.SetInsertPoint(IncomingBB->getTerminator());
1142 Builder.SetCurrentDebugLocation(PHI->getDebugLoc());
1143 LoadInst *EdgeLoad = Builder.CreateLoad(Ty: NewGV->getType(), Ptr: NewGV);
1144 PHI->setIncomingValue(i: I, V: EdgeLoad);
1145 }
1146 continue;
1147 }
1148
1149 Builder.SetInsertPoint(Insn);
1150 Builder.SetCurrentDebugLocation(Insn->getDebugLoc());
1151 LoadInst *Load = Builder.CreateLoad(Ty: NewGV->getType(), Ptr: NewGV);
1152
1153 // The replacement declare target global lives in the default address
1154 // space, whereas the original global may reside in a non-default
1155 // address space. In that case the initial lowering may have
1156 // emitted an addrspacecast that is no longer valid. Replace the
1157 // whole addrspacecast with the load and erase it rather than
1158 // feeding the load back into the (now pointless) cast.
1159 // NOTE: If we end up with replacement declare target globals in
1160 // non-zero AS's the below will need some minor extensions to have the
1161 // option to alter the address space cast to the new address space where
1162 // required rather than just replacing it.
1163 if (auto *ASC = dyn_cast<AddrSpaceCastInst>(Val: Insn)) {
1164 unsigned NewGVAS = NewGV->getType()->getPointerAddressSpace();
1165 assert(NewGVAS == 0 &&
1166 "Non-default address space declare target global");
1167 unsigned OldGVAS = OldGV->getType()->getPointerAddressSpace();
1168 unsigned DestAS = ASC->getType()->getPointerAddressSpace();
1169 if (DestAS == 0 && NewGVAS != OldGVAS) {
1170 ASC->replaceAllUsesWith(V: Load);
1171 ASC->eraseFromParent();
1172 continue;
1173 }
1174 }
1175
1176 Insn->replaceUsesOfWith(From: OldGV, To: Load);
1177 }
1178 }
1179
1180 DeclareTargetGlobalReplacements.clear();
1181}
1182
1183OpenMPIRBuilder::~OpenMPIRBuilder() {
1184 assert(OutlineInfos.empty() && "There must be no outstanding outlinings");
1185}
1186
1187GlobalValue *OpenMPIRBuilder::createGlobalFlag(unsigned Value, StringRef Name) {
1188 IntegerType *I32Ty = Type::getInt32Ty(C&: M.getContext());
1189 auto *GV =
1190 new GlobalVariable(M, I32Ty,
1191 /* isConstant = */ true, GlobalValue::WeakODRLinkage,
1192 ConstantInt::get(Ty: I32Ty, V: Value), Name);
1193 GV->setVisibility(GlobalValue::HiddenVisibility);
1194
1195 return GV;
1196}
1197
1198void OpenMPIRBuilder::emitUsed(StringRef Name, ArrayRef<WeakTrackingVH> List) {
1199 if (List.empty())
1200 return;
1201
1202 // Convert List to what ConstantArray needs.
1203 SmallVector<Constant *, 8> UsedArray;
1204 UsedArray.resize(N: List.size());
1205 for (unsigned I = 0, E = List.size(); I != E; ++I)
1206 UsedArray[I] = ConstantExpr::getPointerBitCastOrAddrSpaceCast(
1207 C: cast<Constant>(Val: &*List[I]), Ty: Builder.getPtrTy());
1208
1209 if (UsedArray.empty())
1210 return;
1211 ArrayType *ATy = ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: UsedArray.size());
1212
1213 auto *GV = new GlobalVariable(M, ATy, false, GlobalValue::AppendingLinkage,
1214 ConstantArray::get(T: ATy, V: UsedArray), Name);
1215
1216 GV->setSection("llvm.metadata");
1217}
1218
1219GlobalVariable *
1220OpenMPIRBuilder::emitKernelExecutionMode(StringRef KernelName,
1221 OMPTgtExecModeFlags Mode) {
1222 auto *Int8Ty = Builder.getInt8Ty();
1223 auto *GVMode = new GlobalVariable(
1224 M, Int8Ty, /*isConstant=*/true, GlobalValue::WeakAnyLinkage,
1225 ConstantInt::get(Ty: Int8Ty, V: Mode), Twine(KernelName, "_exec_mode"));
1226 GVMode->setVisibility(GlobalVariable::ProtectedVisibility);
1227 return GVMode;
1228}
1229
1230Constant *OpenMPIRBuilder::getOrCreateIdent(Constant *SrcLocStr,
1231 uint32_t SrcLocStrSize,
1232 IdentFlag LocFlags,
1233 unsigned Reserve2Flags) {
1234 // Enable "C-mode".
1235 LocFlags |= OMP_IDENT_FLAG_KMPC;
1236
1237 Constant *&Ident =
1238 IdentMap[{SrcLocStr, uint64_t(LocFlags) << 31 | Reserve2Flags}];
1239 if (!Ident) {
1240 Constant *I32Null = ConstantInt::getNullValue(Ty: Int32);
1241 Constant *IdentData[] = {I32Null,
1242 ConstantInt::get(Ty: Int32, V: uint32_t(LocFlags)),
1243 ConstantInt::get(Ty: Int32, V: Reserve2Flags),
1244 ConstantInt::get(Ty: Int32, V: SrcLocStrSize), SrcLocStr};
1245
1246 size_t SrcLocStrArgIdx = 4;
1247 if (OpenMPIRBuilder::Ident->getElementType(N: SrcLocStrArgIdx)
1248 ->getPointerAddressSpace() !=
1249 IdentData[SrcLocStrArgIdx]->getType()->getPointerAddressSpace())
1250 IdentData[SrcLocStrArgIdx] = ConstantExpr::getAddrSpaceCast(
1251 C: SrcLocStr, Ty: OpenMPIRBuilder::Ident->getElementType(N: SrcLocStrArgIdx));
1252 Constant *Initializer =
1253 ConstantStruct::get(T: OpenMPIRBuilder::Ident, V: IdentData);
1254
1255 // Look for existing encoding of the location + flags, not needed but
1256 // minimizes the difference to the existing solution while we transition.
1257 for (GlobalVariable &GV : M.globals())
1258 if (GV.getValueType() == OpenMPIRBuilder::Ident && GV.hasInitializer())
1259 if (GV.getInitializer() == Initializer)
1260 Ident = &GV;
1261
1262 if (!Ident) {
1263 auto *GV = new GlobalVariable(
1264 M, OpenMPIRBuilder::Ident,
1265 /* isConstant = */ true, GlobalValue::PrivateLinkage, Initializer, "",
1266 nullptr, GlobalValue::NotThreadLocal,
1267 M.getDataLayout().getDefaultGlobalsAddressSpace());
1268 GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
1269 GV->setAlignment(Align(8));
1270 Ident = GV;
1271 }
1272 }
1273
1274 return ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: Ident, Ty: IdentPtr);
1275}
1276
1277Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(StringRef LocStr,
1278 uint32_t &SrcLocStrSize) {
1279 SrcLocStrSize = LocStr.size();
1280 Constant *&SrcLocStr = SrcLocStrMap[LocStr];
1281 if (!SrcLocStr) {
1282 Constant *Initializer =
1283 ConstantDataArray::getString(Context&: M.getContext(), Initializer: LocStr);
1284
1285 // Look for existing encoding of the location, not needed but minimizes the
1286 // difference to the existing solution while we transition.
1287 for (GlobalVariable &GV : M.globals())
1288 if (GV.isConstant() && GV.hasInitializer() &&
1289 GV.getInitializer() == Initializer)
1290 return SrcLocStr = ConstantExpr::getPointerCast(C: &GV, Ty: Int8Ptr);
1291
1292 SrcLocStr = Builder.CreateGlobalString(
1293 Str: LocStr, /*Name=*/"", AddressSpace: M.getDataLayout().getDefaultGlobalsAddressSpace(),
1294 M: &M);
1295 }
1296 return SrcLocStr;
1297}
1298
1299Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(StringRef FunctionName,
1300 StringRef FileName,
1301 unsigned Line, unsigned Column,
1302 uint32_t &SrcLocStrSize) {
1303 SmallString<128> Buffer;
1304 Buffer.push_back(Elt: ';');
1305 Buffer.append(RHS: FileName);
1306 Buffer.push_back(Elt: ';');
1307 Buffer.append(RHS: FunctionName);
1308 Buffer.push_back(Elt: ';');
1309 Buffer.append(RHS: std::to_string(val: Line));
1310 Buffer.push_back(Elt: ';');
1311 Buffer.append(RHS: std::to_string(val: Column));
1312 Buffer.push_back(Elt: ';');
1313 Buffer.push_back(Elt: ';');
1314 return getOrCreateSrcLocStr(LocStr: Buffer.str(), SrcLocStrSize);
1315}
1316
1317Constant *
1318OpenMPIRBuilder::getOrCreateDefaultSrcLocStr(uint32_t &SrcLocStrSize) {
1319 StringRef UnknownLoc = ";unknown;unknown;0;0;;";
1320 return getOrCreateSrcLocStr(LocStr: UnknownLoc, SrcLocStrSize);
1321}
1322
1323Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(DebugLoc DL,
1324 uint32_t &SrcLocStrSize,
1325 Function *F) {
1326 DILocation *DIL = DL.get();
1327 if (!DIL)
1328 return getOrCreateDefaultSrcLocStr(SrcLocStrSize);
1329 StringRef FileName =
1330 !DIL->getFilename().empty() ? DIL->getFilename() : M.getName();
1331 StringRef Function = DIL->getScope()->getSubprogram()->getName();
1332 if (Function.empty() && F)
1333 Function = F->getName();
1334 return getOrCreateSrcLocStr(FunctionName: Function, FileName, Line: DIL->getLine(),
1335 Column: DIL->getColumn(), SrcLocStrSize);
1336}
1337
1338Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(const LocationDescription &Loc,
1339 uint32_t &SrcLocStrSize) {
1340 return getOrCreateSrcLocStr(DL: Loc.DL, SrcLocStrSize,
1341 F: Loc.IP.getBlock()->getParent());
1342}
1343
1344Value *OpenMPIRBuilder::getOrCreateThreadID(Value *Ident) {
1345 return createRuntimeFunctionCall(
1346 Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_global_thread_num), Args: Ident,
1347 Name: "omp_global_thread_num");
1348}
1349
1350OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTargetInReduction(
1351 const LocationDescription &Loc, ArrayRef<Value *> OrigPtrs,
1352 ArrayRef<Type *> ResultPtrTys,
1353 function_ref<void(unsigned, Value *)> MapPrivateCB) {
1354 assert(OrigPtrs.size() == ResultPtrTys.size() &&
1355 "expected one result pointer type per in_reduction item");
1356 if (!updateToLocation(Loc))
1357 return Loc.IP;
1358 if (OrigPtrs.empty())
1359 return Builder.saveIP();
1360
1361 // Compute the executing thread's gtid once for the whole target body and
1362 // reuse it for every in_reduction lookup, so a target with several
1363 // in_reduction items does not emit a redundant __kmpc_global_thread_num per
1364 // item.
1365 uint32_t SrcLocStrSize;
1366 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1367 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
1368 Value *Gtid = getOrCreateThreadID(Ident);
1369
1370 // The runtime entry point takes (and returns) a generic, default-address-
1371 // space `ptr`. A NULL descriptor makes the runtime walk the enclosing
1372 // taskgroups to find the matching task_reduction registration for the item.
1373 Type *PtrTy = PointerType::getUnqual(C&: M.getContext());
1374 Value *NullDesc = ConstantPointerNull::get(T: PtrTy);
1375 FunctionCallee GetThData =
1376 getOrCreateRuntimeFunction(M, FnID: OMPRTL___kmpc_task_reduction_get_th_data);
1377
1378 for (unsigned Idx = 0; Idx < OrigPtrs.size(); ++Idx) {
1379 // Normalize a non-default-address-space original pointer to the generic
1380 // address space before the call.
1381 Value *OrigPtr = OrigPtrs[Idx];
1382 if (auto *OrigPtrTy = dyn_cast<PointerType>(Val: OrigPtr->getType());
1383 OrigPtrTy && OrigPtrTy->getAddressSpace() != 0)
1384 OrigPtr = Builder.CreateAddrSpaceCast(V: OrigPtr, DestTy: PtrTy);
1385
1386 Value *Priv = Builder.CreateCall(Callee: GetThData, Args: {Gtid, NullDesc, OrigPtr},
1387 Name: "omp.inred.priv");
1388
1389 // Cast the returned private pointer back to the requested address space
1390 // when it differs.
1391 if (auto *ResPtrTy = dyn_cast<PointerType>(Val: ResultPtrTys[Idx]);
1392 ResPtrTy && ResPtrTy->getAddressSpace() != 0)
1393 Priv = Builder.CreateAddrSpaceCast(V: Priv, DestTy: ResultPtrTys[Idx]);
1394
1395 MapPrivateCB(Idx, Priv);
1396 }
1397 return Builder.saveIP();
1398}
1399
1400OpenMPIRBuilder::InsertPointOrErrorTy
1401OpenMPIRBuilder::createBarrier(const LocationDescription &Loc, Directive Kind,
1402 bool ForceSimpleCall, bool CheckCancelFlag) {
1403 if (!updateToLocation(Loc))
1404 return Loc.IP;
1405
1406 // Build call __kmpc_cancel_barrier(loc, thread_id) or
1407 // __kmpc_barrier(loc, thread_id);
1408
1409 IdentFlag BarrierLocFlags;
1410 switch (Kind) {
1411 case OMPD_for:
1412 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_FOR;
1413 break;
1414 case OMPD_sections:
1415 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_SECTIONS;
1416 break;
1417 case OMPD_single:
1418 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_SINGLE;
1419 break;
1420 case OMPD_barrier:
1421 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_EXPL;
1422 break;
1423 default:
1424 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL;
1425 break;
1426 }
1427
1428 uint32_t SrcLocStrSize;
1429 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1430 Value *Args[] = {
1431 getOrCreateIdent(SrcLocStr, SrcLocStrSize, LocFlags: BarrierLocFlags),
1432 getOrCreateThreadID(Ident: getOrCreateIdent(SrcLocStr, SrcLocStrSize))};
1433
1434 // If we are in a cancellable parallel region, barriers are cancellation
1435 // points.
1436 // TODO: Check why we would force simple calls or to ignore the cancel flag.
1437 bool UseCancelBarrier =
1438 !ForceSimpleCall && isLastFinalizationInfoCancellable(DK: OMPD_parallel);
1439
1440 Value *Result = createRuntimeFunctionCall(
1441 Callee: getOrCreateRuntimeFunctionPtr(FnID: UseCancelBarrier
1442 ? OMPRTL___kmpc_cancel_barrier
1443 : OMPRTL___kmpc_barrier),
1444 Args);
1445
1446 if (UseCancelBarrier && CheckCancelFlag)
1447 if (Error Err = emitCancelationCheckImpl(CancelFlag: Result, CanceledDirective: OMPD_parallel))
1448 return Err;
1449
1450 return Builder.saveIP();
1451}
1452
1453OpenMPIRBuilder::InsertPointOrErrorTy
1454OpenMPIRBuilder::createCancel(const LocationDescription &Loc,
1455 Value *IfCondition,
1456 omp::Directive CanceledDirective) {
1457 if (!updateToLocation(Loc))
1458 return Loc.IP;
1459
1460 // LLVM utilities like blocks with terminators.
1461 auto *UI = Builder.CreateUnreachable();
1462
1463 Instruction *ThenTI = UI, *ElseTI = nullptr;
1464 if (IfCondition) {
1465 SplitBlockAndInsertIfThenElse(Cond: IfCondition, SplitBefore: UI, ThenTerm: &ThenTI, ElseTerm: &ElseTI);
1466
1467 // Even if the if condition evaluates to false, this should count as a
1468 // cancellation point
1469 Builder.SetInsertPoint(ElseTI);
1470 auto ElseIP = Builder.saveIP();
1471
1472 InsertPointOrErrorTy IPOrErr = createCancellationPoint(
1473 Loc: LocationDescription{ElseIP, Loc.DL}, CanceledDirective);
1474 if (!IPOrErr)
1475 return IPOrErr;
1476 }
1477
1478 Builder.SetInsertPoint(ThenTI);
1479
1480 Value *CancelKind = nullptr;
1481 switch (CanceledDirective) {
1482#define OMP_CANCEL_KIND(Enum, Str, DirectiveEnum, Value) \
1483 case DirectiveEnum: \
1484 CancelKind = Builder.getInt32(Value); \
1485 break;
1486#include "llvm/Frontend/OpenMP/OMPKinds.def"
1487 default:
1488 llvm_unreachable("Unknown cancel kind!");
1489 }
1490
1491 uint32_t SrcLocStrSize;
1492 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1493 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
1494 Value *Args[] = {Ident, getOrCreateThreadID(Ident), CancelKind};
1495 Value *Result = createRuntimeFunctionCall(
1496 Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_cancel), Args);
1497
1498 // The actual cancel logic is shared with others, e.g., cancel_barriers.
1499 if (Error Err = emitCancelationCheckImpl(CancelFlag: Result, CanceledDirective))
1500 return Err;
1501
1502 // Update the insertion point and remove the terminator we introduced.
1503 Builder.SetInsertPoint(UI->getParent());
1504 UI->eraseFromParent();
1505
1506 return Builder.saveIP();
1507}
1508
1509OpenMPIRBuilder::InsertPointOrErrorTy
1510OpenMPIRBuilder::createCancellationPoint(const LocationDescription &Loc,
1511 omp::Directive CanceledDirective) {
1512 if (!updateToLocation(Loc))
1513 return Loc.IP;
1514
1515 // LLVM utilities like blocks with terminators.
1516 auto *UI = Builder.CreateUnreachable();
1517 Builder.SetInsertPoint(UI);
1518
1519 Value *CancelKind = nullptr;
1520 switch (CanceledDirective) {
1521#define OMP_CANCEL_KIND(Enum, Str, DirectiveEnum, Value) \
1522 case DirectiveEnum: \
1523 CancelKind = Builder.getInt32(Value); \
1524 break;
1525#include "llvm/Frontend/OpenMP/OMPKinds.def"
1526 default:
1527 llvm_unreachable("Unknown cancel kind!");
1528 }
1529
1530 uint32_t SrcLocStrSize;
1531 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1532 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
1533 Value *Args[] = {Ident, getOrCreateThreadID(Ident), CancelKind};
1534 Value *Result = createRuntimeFunctionCall(
1535 Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_cancellationpoint), Args);
1536
1537 // The actual cancel logic is shared with others, e.g., cancel_barriers.
1538 if (Error Err = emitCancelationCheckImpl(CancelFlag: Result, CanceledDirective))
1539 return Err;
1540
1541 // Update the insertion point and remove the terminator we introduced.
1542 Builder.SetInsertPoint(UI->getParent());
1543 UI->eraseFromParent();
1544
1545 return Builder.saveIP();
1546}
1547
1548OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitTargetKernel(
1549 const LocationDescription &Loc, InsertPointTy AllocaIP, Value *&Return,
1550 Value *Ident, Value *DeviceID, Value *NumTeams, Value *NumThreads,
1551 Value *HostPtr, ArrayRef<Value *> KernelArgs) {
1552 if (!updateToLocation(Loc))
1553 return Loc.IP;
1554
1555 Builder.restoreIP(IP: AllocaIP);
1556 auto *KernelArgsPtr =
1557 Builder.CreateAlloca(Ty: OpenMPIRBuilder::KernelArgs, ArraySize: nullptr, Name: "kernel_args");
1558 updateToLocation(Loc);
1559
1560 for (unsigned I = 0, Size = KernelArgs.size(); I != Size; ++I) {
1561 llvm::Value *Arg =
1562 Builder.CreateStructGEP(Ty: OpenMPIRBuilder::KernelArgs, Ptr: KernelArgsPtr, Idx: I);
1563 Builder.CreateAlignedStore(
1564 Val: KernelArgs[I], Ptr: Arg,
1565 Align: M.getDataLayout().getPrefTypeAlign(Ty: KernelArgs[I]->getType()));
1566 }
1567
1568 SmallVector<Value *> OffloadingArgs{Ident, DeviceID, NumTeams,
1569 NumThreads, HostPtr, KernelArgsPtr};
1570
1571 Return = createRuntimeFunctionCall(
1572 Callee: getOrCreateRuntimeFunction(M, FnID: OMPRTL___tgt_target_kernel),
1573 Args: OffloadingArgs);
1574
1575 return Builder.saveIP();
1576}
1577
1578OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitKernelLaunch(
1579 const LocationDescription &Loc, Value *OutlinedFnID,
1580 EmitFallbackCallbackTy EmitTargetCallFallbackCB, TargetKernelArgs &Args,
1581 Value *DeviceID, Value *RTLoc, InsertPointTy AllocaIP) {
1582
1583 if (!updateToLocation(Loc))
1584 return Loc.IP;
1585
1586 // On top of the arrays that were filled up, the target offloading call
1587 // takes as arguments the device id as well as the host pointer. The host
1588 // pointer is used by the runtime library to identify the current target
1589 // region, so it only has to be unique and not necessarily point to
1590 // anything. It could be the pointer to the outlined function that
1591 // implements the target region, but we aren't using that so that the
1592 // compiler doesn't need to keep that, and could therefore inline the host
1593 // function if proven worthwhile during optimization.
1594
1595 // From this point on, we need to have an ID of the target region defined.
1596 assert(OutlinedFnID && "Invalid outlined function ID!");
1597 (void)OutlinedFnID;
1598
1599 // Return value of the runtime offloading call.
1600 Value *Return = nullptr;
1601
1602 // Arguments for the target kernel.
1603 SmallVector<Value *> ArgsVector;
1604 getKernelArgsVector(KernelArgs&: Args, Builder, ArgsVector);
1605
1606 // The target region is an outlined function launched by the runtime
1607 // via calls to __tgt_target_kernel().
1608 //
1609 // Note that on the host and CPU targets, the runtime implementation of
1610 // these calls simply call the outlined function without forking threads.
1611 // The outlined functions themselves have runtime calls to
1612 // __kmpc_fork_teams() and __kmpc_fork() for this purpose, codegen'd by
1613 // the compiler in emitTeamsCall() and emitParallelCall().
1614 //
1615 // In contrast, on the NVPTX target, the implementation of
1616 // __tgt_target_teams() launches a GPU kernel with the requested number
1617 // of teams and threads so no additional calls to the runtime are required.
1618 // Check the error code and execute the host version if required.
1619 Builder.restoreIP(IP: emitTargetKernel(
1620 Loc: Builder, AllocaIP, Return, Ident: RTLoc, DeviceID, NumTeams: Args.NumTeams.front(),
1621 NumThreads: Args.NumThreads.front(), HostPtr: OutlinedFnID, KernelArgs: ArgsVector));
1622
1623 BasicBlock *OffloadFailedBlock =
1624 BasicBlock::Create(Context&: Builder.getContext(), Name: "omp_offload.failed");
1625 BasicBlock *OffloadContBlock =
1626 BasicBlock::Create(Context&: Builder.getContext(), Name: "omp_offload.cont");
1627 Value *Failed = Builder.CreateIsNotNull(Arg: Return);
1628 Builder.CreateCondBr(Cond: Failed, True: OffloadFailedBlock, False: OffloadContBlock);
1629
1630 auto CurFn = Builder.GetInsertBlock()->getParent();
1631 emitBlock(BB: OffloadFailedBlock, CurFn);
1632 InsertPointOrErrorTy AfterIP = EmitTargetCallFallbackCB(Builder.saveIP());
1633 if (!AfterIP)
1634 return AfterIP.takeError();
1635 Builder.restoreIP(IP: *AfterIP);
1636 emitBranch(Target: OffloadContBlock);
1637 emitBlock(BB: OffloadContBlock, CurFn, /*IsFinished=*/true);
1638 return Builder.saveIP();
1639}
1640
1641Error OpenMPIRBuilder::emitCancelationCheckImpl(
1642 Value *CancelFlag, omp::Directive CanceledDirective) {
1643 assert(isLastFinalizationInfoCancellable(CanceledDirective) &&
1644 "Unexpected cancellation!");
1645
1646 // For a cancel barrier we create two new blocks.
1647 BasicBlock *BB = Builder.GetInsertBlock();
1648 BasicBlock *NonCancellationBlock;
1649 if (Builder.GetInsertPoint() == BB->end()) {
1650 // TODO: This branch will not be needed once we moved to the
1651 // OpenMPIRBuilder codegen completely.
1652 NonCancellationBlock = BasicBlock::Create(
1653 Context&: BB->getContext(), Name: BB->getName() + ".cont", Parent: BB->getParent());
1654 } else {
1655 NonCancellationBlock = SplitBlock(Old: BB, SplitPt: &*Builder.GetInsertPoint());
1656 BB->getTerminator()->eraseFromParent();
1657 Builder.SetInsertPoint(BB);
1658 }
1659 BasicBlock *CancellationBlock = BasicBlock::Create(
1660 Context&: BB->getContext(), Name: BB->getName() + ".cncl", Parent: BB->getParent());
1661
1662 // Jump to them based on the return value.
1663 Value *Cmp = Builder.CreateIsNull(Arg: CancelFlag);
1664 Builder.CreateCondBr(Cond: Cmp, True: NonCancellationBlock, False: CancellationBlock,
1665 /* TODO weight */ BranchWeights: nullptr, Unpredictable: nullptr);
1666
1667 // From the cancellation block we finalize all variables and go to the
1668 // post finalization block that is known to the FiniCB callback.
1669 auto &FI = FinalizationStack.back();
1670 Expected<BasicBlock *> FiniBBOrErr = FI.getFiniBB(Builder);
1671 if (!FiniBBOrErr)
1672 return FiniBBOrErr.takeError();
1673 Builder.SetInsertPoint(CancellationBlock);
1674 Builder.CreateBr(Dest: *FiniBBOrErr);
1675
1676 // The continuation block is where code generation continues.
1677 Builder.SetInsertPoint(TheBB: NonCancellationBlock, IP: NonCancellationBlock->begin());
1678 return Error::success();
1679}
1680
1681/// Create wrapper function used to gather the outlined function's argument
1682/// structure from a shared buffer and to forward them to it when running in
1683/// Generic mode.
1684///
1685/// The outlined function is expected to receive 2 integer arguments followed by
1686/// an optional pointer argument to an argument structure holding the rest.
1687static Function *createTargetParallelWrapper(OpenMPIRBuilder *OMPIRBuilder,
1688 Function &OutlinedFn) {
1689 size_t NumArgs = OutlinedFn.arg_size();
1690 assert((NumArgs == 2 || NumArgs == 3) &&
1691 "expected a 2-3 argument parallel outlined function");
1692 bool UseArgStruct = NumArgs == 3;
1693
1694 IRBuilder<> &Builder = OMPIRBuilder->Builder;
1695 IRBuilder<>::InsertPointGuard IPG(Builder);
1696 auto *FnTy = FunctionType::get(Result: Builder.getVoidTy(),
1697 Params: {Builder.getInt16Ty(), Builder.getInt32Ty()},
1698 /*isVarArg=*/false);
1699 auto *WrapperFn =
1700 Function::Create(Ty: FnTy, Linkage: GlobalValue::InternalLinkage,
1701 N: OutlinedFn.getName() + ".wrapper", M&: OMPIRBuilder->M);
1702
1703 WrapperFn->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
1704 WrapperFn->addParamAttr(ArgNo: 0, Kind: Attribute::ZExt);
1705 WrapperFn->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
1706
1707 BasicBlock *EntryBB =
1708 BasicBlock::Create(Context&: OMPIRBuilder->M.getContext(), Name: "entry", Parent: WrapperFn);
1709 Builder.SetInsertPoint(EntryBB);
1710
1711 // Allocation.
1712 Value *AddrAlloca = Builder.CreateAlloca(Ty: Builder.getInt32Ty(),
1713 /*ArraySize=*/nullptr, Name: "addr");
1714 AddrAlloca = Builder.CreatePointerBitCastOrAddrSpaceCast(
1715 V: AddrAlloca, DestTy: Builder.getPtrTy(/*AddrSpace=*/0),
1716 Name: AddrAlloca->getName() + ".ascast");
1717
1718 Value *ZeroAlloca = Builder.CreateAlloca(Ty: Builder.getInt32Ty(),
1719 /*ArraySize=*/nullptr, Name: "zero");
1720 ZeroAlloca = Builder.CreatePointerBitCastOrAddrSpaceCast(
1721 V: ZeroAlloca, DestTy: Builder.getPtrTy(/*AddrSpace=*/0),
1722 Name: ZeroAlloca->getName() + ".ascast");
1723
1724 Value *ArgsAlloca = nullptr;
1725 if (UseArgStruct) {
1726 ArgsAlloca = Builder.CreateAlloca(Ty: Builder.getPtrTy(),
1727 /*ArraySize=*/nullptr, Name: "global_args");
1728 ArgsAlloca = Builder.CreatePointerBitCastOrAddrSpaceCast(
1729 V: ArgsAlloca, DestTy: Builder.getPtrTy(/*AddrSpace=*/0),
1730 Name: ArgsAlloca->getName() + ".ascast");
1731 }
1732
1733 // Initialization.
1734 Builder.CreateStore(Val: WrapperFn->getArg(i: 1), Ptr: AddrAlloca);
1735 Builder.CreateStore(Val: Builder.getInt32(C: 0), Ptr: ZeroAlloca);
1736 if (UseArgStruct) {
1737 Builder.CreateCall(
1738 Callee: OMPIRBuilder->getOrCreateRuntimeFunctionPtr(
1739 FnID: llvm::omp::RuntimeFunction::OMPRTL___kmpc_get_shared_variables),
1740 Args: {ArgsAlloca});
1741 }
1742
1743 SmallVector<Value *, 3> Args{AddrAlloca, ZeroAlloca};
1744
1745 // Load structArg from global_args.
1746 if (UseArgStruct) {
1747 Value *StructArg = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ArgsAlloca);
1748 StructArg = Builder.CreateInBoundsGEP(Ty: Builder.getPtrTy(), Ptr: StructArg,
1749 IdxList: {Builder.getInt64(C: 0)});
1750 StructArg = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: StructArg, Name: "structArg");
1751 Args.push_back(Elt: StructArg);
1752 }
1753
1754 // Call the outlined function holding the parallel body.
1755 Builder.CreateCall(Callee: &OutlinedFn, Args);
1756 Builder.CreateRetVoid();
1757
1758 return WrapperFn;
1759}
1760
1761// Callback used to create OpenMP runtime calls to support
1762// omp parallel clause for the device.
1763// We need to use this callback to replace call to the OutlinedFn in OuterFn
1764// by the call to the OpenMP DeviceRTL runtime function (kmpc_parallel_60)
1765static void targetParallelCallback(
1766 OpenMPIRBuilder *OMPIRBuilder, Function &OutlinedFn, Function *OuterFn,
1767 BasicBlock *OuterAllocaBB, Value *Ident, Value *IfCondition,
1768 Value *NumThreads, Instruction *PrivTID, AllocaInst *PrivTIDAddr,
1769 Value *ThreadID, const SmallVector<Instruction *, 4> &ToBeDeleted) {
1770 assert(OutlinedFn.arg_size() >= 2 &&
1771 "Expected at least tid and bounded tid as arguments");
1772 unsigned NumCapturedVars = OutlinedFn.arg_size() - /* tid & bounded tid */ 2;
1773
1774 // Add some known attributes.
1775 IRBuilder<> &Builder = OMPIRBuilder->Builder;
1776 OutlinedFn.addParamAttr(ArgNo: 0, Kind: Attribute::NoAlias);
1777 OutlinedFn.addParamAttr(ArgNo: 1, Kind: Attribute::NoAlias);
1778 OutlinedFn.addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
1779 OutlinedFn.addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
1780 OutlinedFn.addFnAttr(Kind: Attribute::NoUnwind);
1781
1782 CallInst *CI = cast<CallInst>(Val: OutlinedFn.user_back());
1783 assert(CI && "Expected call instruction to outlined function");
1784 CI->getParent()->setName("omp_parallel");
1785
1786 Builder.SetInsertPoint(CI);
1787 Type *PtrTy = OMPIRBuilder->VoidPtr;
1788
1789 // Add alloca for kernel args
1790 OpenMPIRBuilder ::InsertPointTy CurrentIP = Builder.saveIP();
1791 Builder.SetInsertPoint(TheBB: OuterAllocaBB, IP: OuterAllocaBB->getFirstInsertionPt());
1792 AllocaInst *ArgsAlloca =
1793 Builder.CreateAlloca(Ty: ArrayType::get(ElementType: PtrTy, NumElements: NumCapturedVars));
1794 Value *Args = ArgsAlloca;
1795 // Add address space cast if array for storing arguments is not allocated
1796 // in address space 0
1797 if (ArgsAlloca->getAddressSpace())
1798 Args = Builder.CreatePointerCast(V: ArgsAlloca, DestTy: PtrTy);
1799 Builder.restoreIP(IP: CurrentIP);
1800
1801 // Store captured vars which are used by kmpc_parallel_60
1802 for (unsigned Idx = 0; Idx < NumCapturedVars; Idx++) {
1803 Value *V = *(CI->arg_begin() + 2 + Idx);
1804 Value *StoreAddress = Builder.CreateConstInBoundsGEP2_64(
1805 Ty: ArrayType::get(ElementType: PtrTy, NumElements: NumCapturedVars), Ptr: Args, Idx0: 0, Idx1: Idx);
1806 Builder.CreateStore(Val: V, Ptr: StoreAddress);
1807 }
1808
1809 Value *Cond =
1810 IfCondition ? Builder.CreateSExtOrTrunc(V: IfCondition, DestTy: OMPIRBuilder->Int32)
1811 : Builder.getInt32(C: 1);
1812 Value *NumThreadsArg =
1813 NumThreads ? Builder.CreateZExtOrTrunc(V: NumThreads, DestTy: OMPIRBuilder->Int32)
1814 : Builder.getInt32(C: -1);
1815
1816 // If this is not a Generic kernel, we can skip generating the wrapper.
1817 Value *WrapperFn;
1818 if (isGenericKernel(Fn&: *OuterFn))
1819 WrapperFn = createTargetParallelWrapper(OMPIRBuilder, OutlinedFn);
1820 else
1821 WrapperFn = Constant::getNullValue(Ty: PtrTy);
1822
1823 // Build kmpc_parallel_60 call
1824 Value *Parallel60CallArgs[] = {
1825 /* identifier*/ Ident,
1826 /* global thread num*/ ThreadID,
1827 /* if expression */ Cond,
1828 /* number of threads */ NumThreadsArg,
1829 /* Proc bind */ Builder.getInt32(C: -1),
1830 /* outlined function */ &OutlinedFn,
1831 /* wrapper function */ WrapperFn,
1832 /* arguments of the outlined funciton*/ Args,
1833 /* number of arguments */ Builder.getInt64(C: NumCapturedVars),
1834 /* strict for number of threads */ Builder.getInt32(C: 0)};
1835
1836 FunctionCallee RTLFn =
1837 OMPIRBuilder->getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_parallel_60);
1838
1839 OMPIRBuilder->createRuntimeFunctionCall(Callee: RTLFn, Args: Parallel60CallArgs);
1840
1841 LLVM_DEBUG(dbgs() << "With kmpc_parallel_60 placed: "
1842 << *Builder.GetInsertBlock()->getParent() << "\n");
1843
1844 // Initialize the local TID stack location with the argument value.
1845 Builder.SetInsertPoint(PrivTID);
1846 Function::arg_iterator OutlinedAI = OutlinedFn.arg_begin();
1847 Builder.CreateStore(Val: Builder.CreateLoad(Ty: OMPIRBuilder->Int32, Ptr: OutlinedAI),
1848 Ptr: PrivTIDAddr);
1849
1850 // Remove redundant call to the outlined function.
1851 CI->eraseFromParent();
1852
1853 for (Instruction *I : ToBeDeleted) {
1854 I->eraseFromParent();
1855 }
1856}
1857
1858// Callback used to create OpenMP runtime calls to support
1859// omp parallel clause for the host.
1860// We need to use this callback to replace call to the OutlinedFn in OuterFn
1861// by the call to the OpenMP host runtime function ( __kmpc_fork_call[_if])
1862static void
1863hostParallelCallback(OpenMPIRBuilder *OMPIRBuilder, Function &OutlinedFn,
1864 Function *OuterFn, Value *Ident, Value *IfCondition,
1865 Instruction *PrivTID, AllocaInst *PrivTIDAddr,
1866 const SmallVector<Instruction *, 4> &ToBeDeleted) {
1867 IRBuilder<> &Builder = OMPIRBuilder->Builder;
1868 FunctionCallee RTLFn;
1869 if (IfCondition) {
1870 RTLFn =
1871 OMPIRBuilder->getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_fork_call_if);
1872 } else {
1873 RTLFn =
1874 OMPIRBuilder->getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_fork_call);
1875 }
1876 if (auto *F = dyn_cast<Function>(Val: RTLFn.getCallee())) {
1877 if (!F->hasMetadata(KindID: LLVMContext::MD_callback)) {
1878 LLVMContext &Ctx = F->getContext();
1879 MDBuilder MDB(Ctx);
1880 // Annotate the callback behavior of the __kmpc_fork_call:
1881 // - The callback callee is argument number 2 (microtask).
1882 // - The first two arguments of the callback callee are unknown (-1).
1883 // - All variadic arguments to the __kmpc_fork_call are passed to the
1884 // callback callee.
1885 F->addMetadata(KindID: LLVMContext::MD_callback,
1886 MD&: *MDNode::get(Context&: Ctx, MDs: {MDB.createCallbackEncoding(
1887 CalleeArgNo: 2, Arguments: {-1, -1},
1888 /* VarArgsArePassed */ true)}));
1889 }
1890 }
1891 // Add some known attributes.
1892 OutlinedFn.addParamAttr(ArgNo: 0, Kind: Attribute::NoAlias);
1893 OutlinedFn.addParamAttr(ArgNo: 1, Kind: Attribute::NoAlias);
1894 OutlinedFn.addFnAttr(Kind: Attribute::NoUnwind);
1895
1896 assert(OutlinedFn.arg_size() >= 2 &&
1897 "Expected at least tid and bounded tid as arguments");
1898 unsigned NumCapturedVars = OutlinedFn.arg_size() - /* tid & bounded tid */ 2;
1899
1900 CallInst *CI = cast<CallInst>(Val: OutlinedFn.user_back());
1901 CI->getParent()->setName("omp_parallel");
1902 Builder.SetInsertPoint(CI);
1903
1904 // Build call __kmpc_fork_call[_if](Ident, n, microtask, var1, .., varn);
1905 Value *ForkCallArgs[] = {Ident, Builder.getInt32(C: NumCapturedVars),
1906 &OutlinedFn};
1907
1908 SmallVector<Value *, 16> RealArgs;
1909 RealArgs.append(in_start: std::begin(arr&: ForkCallArgs), in_end: std::end(arr&: ForkCallArgs));
1910 if (IfCondition) {
1911 Value *Cond = Builder.CreateSExtOrTrunc(V: IfCondition, DestTy: OMPIRBuilder->Int32);
1912 RealArgs.push_back(Elt: Cond);
1913 }
1914 RealArgs.append(in_start: CI->arg_begin() + /* tid & bound tid */ 2, in_end: CI->arg_end());
1915
1916 // __kmpc_fork_call_if always expects a void ptr as the last argument
1917 // If there are no arguments, pass a null pointer.
1918 auto PtrTy = OMPIRBuilder->VoidPtr;
1919 if (IfCondition && NumCapturedVars == 0) {
1920 Value *NullPtrValue = Constant::getNullValue(Ty: PtrTy);
1921 RealArgs.push_back(Elt: NullPtrValue);
1922 }
1923
1924 OMPIRBuilder->createRuntimeFunctionCall(Callee: RTLFn, Args: RealArgs);
1925
1926 LLVM_DEBUG(dbgs() << "With fork_call placed: "
1927 << *Builder.GetInsertBlock()->getParent() << "\n");
1928
1929 // Initialize the local TID stack location with the argument value.
1930 Builder.SetInsertPoint(PrivTID);
1931 Function::arg_iterator OutlinedAI = OutlinedFn.arg_begin();
1932 Builder.CreateStore(Val: Builder.CreateLoad(Ty: OMPIRBuilder->Int32, Ptr: OutlinedAI),
1933 Ptr: PrivTIDAddr);
1934
1935 // Remove redundant call to the outlined function.
1936 CI->eraseFromParent();
1937
1938 for (Instruction *I : ToBeDeleted) {
1939 I->eraseFromParent();
1940 }
1941}
1942
1943OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createParallel(
1944 const LocationDescription &Loc, InsertPointTy OuterAllocIP,
1945 ArrayRef<BasicBlock *> OuterDeallocBlocks, BodyGenCallbackTy BodyGenCB,
1946 PrivatizeCallbackTy PrivCB, FinalizeCallbackTy FiniCB, Value *IfCondition,
1947 Value *NumThreads, omp::ProcBindKind ProcBind, bool IsCancellable) {
1948 assert(!isConflictIP(Loc.IP, OuterAllocIP) && "IPs must not be ambiguous");
1949
1950 if (!updateToLocation(Loc))
1951 return Loc.IP;
1952
1953 uint32_t SrcLocStrSize;
1954 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1955 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
1956 const bool NeedThreadID = NumThreads || Config.isTargetDevice() ||
1957 (ProcBind != OMP_PROC_BIND_default);
1958 Value *ThreadID = NeedThreadID ? getOrCreateThreadID(Ident) : nullptr;
1959 // If we generate code for the target device, we need to allocate
1960 // struct for aggregate params in the device default alloca address space.
1961 // OpenMP runtime requires that the params of the extracted functions are
1962 // passed as zero address space pointers. This flag ensures that extracted
1963 // function arguments are declared in zero address space
1964 bool ArgsInZeroAddressSpace = Config.isTargetDevice();
1965
1966 // Build call __kmpc_push_num_threads(&Ident, global_tid, num_threads)
1967 // only if we compile for host side.
1968 if (NumThreads && !Config.isTargetDevice()) {
1969 Value *Args[] = {
1970 Ident, ThreadID,
1971 Builder.CreateIntCast(V: NumThreads, DestTy: Int32, /*isSigned*/ false)};
1972 createRuntimeFunctionCall(
1973 Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_push_num_threads), Args);
1974 }
1975
1976 if (ProcBind != OMP_PROC_BIND_default) {
1977 // Build call __kmpc_push_proc_bind(&Ident, global_tid, proc_bind)
1978 Value *Args[] = {
1979 Ident, ThreadID,
1980 ConstantInt::get(Ty: Int32, V: unsigned(ProcBind), /*isSigned=*/IsSigned: true)};
1981 createRuntimeFunctionCall(
1982 Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_push_proc_bind), Args);
1983 }
1984
1985 BasicBlock *InsertBB = Builder.GetInsertBlock();
1986 Function *OuterFn = InsertBB->getParent();
1987
1988 // Save the outer alloca block because the insertion iterator may get
1989 // invalidated and we still need this later.
1990 BasicBlock *OuterAllocaBlock = OuterAllocIP.getBlock();
1991
1992 // Vector to remember instructions we used only during the modeling but which
1993 // we want to delete at the end.
1994 SmallVector<Instruction *, 4> ToBeDeleted;
1995
1996 // Change the location to the outer alloca insertion point to create and
1997 // initialize the allocas we pass into the parallel region.
1998 InsertPointTy NewOuter(OuterAllocaBlock, OuterAllocaBlock->begin());
1999 Builder.restoreIP(IP: NewOuter);
2000 AllocaInst *TIDAddrAlloca = Builder.CreateAlloca(Ty: Int32, ArraySize: nullptr, Name: "tid.addr");
2001 AllocaInst *ZeroAddrAlloca =
2002 Builder.CreateAlloca(Ty: Int32, ArraySize: nullptr, Name: "zero.addr");
2003 Instruction *TIDAddr = TIDAddrAlloca;
2004 Instruction *ZeroAddr = ZeroAddrAlloca;
2005 if (ArgsInZeroAddressSpace && M.getDataLayout().getAllocaAddrSpace() != 0) {
2006 // Add additional casts to enforce pointers in zero address space
2007 TIDAddr = new AddrSpaceCastInst(
2008 TIDAddrAlloca, PointerType ::get(C&: M.getContext(), AddressSpace: 0), "tid.addr.ascast");
2009 TIDAddr->insertAfter(InsertPos: TIDAddrAlloca->getIterator());
2010 ToBeDeleted.push_back(Elt: TIDAddr);
2011 ZeroAddr = new AddrSpaceCastInst(ZeroAddrAlloca,
2012 PointerType ::get(C&: M.getContext(), AddressSpace: 0),
2013 "zero.addr.ascast");
2014 ZeroAddr->insertAfter(InsertPos: ZeroAddrAlloca->getIterator());
2015 ToBeDeleted.push_back(Elt: ZeroAddr);
2016 }
2017
2018 // We only need TIDAddr and ZeroAddr for modeling purposes to get the
2019 // associated arguments in the outlined function, so we delete them later.
2020 ToBeDeleted.push_back(Elt: TIDAddrAlloca);
2021 ToBeDeleted.push_back(Elt: ZeroAddrAlloca);
2022
2023 // Create an artificial insertion point that will also ensure the blocks we
2024 // are about to split are not degenerated.
2025 auto *UI = new UnreachableInst(Builder.getContext(), InsertBB);
2026
2027 BasicBlock *EntryBB = UI->getParent();
2028 BasicBlock *PRegEntryBB = EntryBB->splitBasicBlock(I: UI, BBName: "omp.par.entry");
2029 BasicBlock *PRegBodyBB = PRegEntryBB->splitBasicBlock(I: UI, BBName: "omp.par.region");
2030 BasicBlock *PRegPreFiniBB =
2031 PRegBodyBB->splitBasicBlock(I: UI, BBName: "omp.par.pre_finalize");
2032 BasicBlock *PRegExitBB = PRegPreFiniBB->splitBasicBlock(I: UI, BBName: "omp.par.exit");
2033
2034 auto FiniCBWrapper = [&](InsertPointTy IP) {
2035 // Hide "open-ended" blocks from the given FiniCB by setting the right jump
2036 // target to the region exit block.
2037 if (IP.getBlock()->end() == IP.getPoint()) {
2038 IRBuilder<>::InsertPointGuard IPG(Builder);
2039 Builder.restoreIP(IP);
2040 Instruction *I = Builder.CreateBr(Dest: PRegExitBB);
2041 IP = InsertPointTy(I->getParent(), I->getIterator());
2042 }
2043 assert(IP.getBlock()->getTerminator()->getNumSuccessors() == 1 &&
2044 IP.getBlock()->getTerminator()->getSuccessor(0) == PRegExitBB &&
2045 "Unexpected insertion point for finalization call!");
2046 return FiniCB(IP);
2047 };
2048
2049 FinalizationStack.push_back(Elt: {FiniCBWrapper, OMPD_parallel, IsCancellable});
2050
2051 // Generate the privatization allocas in the block that will become the entry
2052 // of the outlined function.
2053 Builder.SetInsertPoint(PRegEntryBB->getTerminator());
2054 InsertPointTy InnerAllocaIP = Builder.saveIP();
2055
2056 AllocaInst *PrivTIDAddr =
2057 Builder.CreateAlloca(Ty: Int32, ArraySize: nullptr, Name: "tid.addr.local");
2058 Instruction *PrivTID = Builder.CreateLoad(Ty: Int32, Ptr: PrivTIDAddr, Name: "tid");
2059
2060 // Add some fake uses for OpenMP provided arguments.
2061 ToBeDeleted.push_back(Elt: Builder.CreateLoad(Ty: Int32, Ptr: TIDAddr, Name: "tid.addr.use"));
2062 Instruction *ZeroAddrUse =
2063 Builder.CreateLoad(Ty: Int32, Ptr: ZeroAddr, Name: "zero.addr.use");
2064 ToBeDeleted.push_back(Elt: ZeroAddrUse);
2065
2066 // EntryBB
2067 // |
2068 // V
2069 // PRegionEntryBB <- Privatization allocas are placed here.
2070 // |
2071 // V
2072 // PRegionBodyBB <- BodeGen is invoked here.
2073 // |
2074 // V
2075 // PRegPreFiniBB <- The block we will start finalization from.
2076 // |
2077 // V
2078 // PRegionExitBB <- A common exit to simplify block collection.
2079 //
2080
2081 LLVM_DEBUG(dbgs() << "Before body codegen: " << *OuterFn << "\n");
2082
2083 // Let the caller create the body.
2084 assert(BodyGenCB && "Expected body generation callback!");
2085 InsertPointTy CodeGenIP(PRegBodyBB, PRegBodyBB->begin());
2086 if (Error Err = BodyGenCB(InnerAllocaIP, CodeGenIP, PRegExitBB))
2087 return Err;
2088
2089 LLVM_DEBUG(dbgs() << "After body codegen: " << *OuterFn << "\n");
2090
2091 // If OuterFn is a Generic kernel, we need to use device shared memory to
2092 // allocate argument structures. Otherwise, we use stack allocations as usual.
2093 bool UsesDeviceSharedMemory =
2094 Config.isTargetDevice() && isGenericKernel(Fn&: *OuterFn);
2095 std::unique_ptr<OutlineInfo> OI =
2096 UsesDeviceSharedMemory
2097 ? std::make_unique<DeviceSharedMemOutlineInfo>(args&: *this)
2098 : std::make_unique<OutlineInfo>();
2099
2100 if (Config.isTargetDevice()) {
2101 // Generate OpenMP target specific runtime call
2102 OI->PostOutlineCB = [=, ToBeDeletedVec =
2103 std::move(ToBeDeleted)](Function &OutlinedFn) {
2104 targetParallelCallback(OMPIRBuilder: this, OutlinedFn, OuterFn, OuterAllocaBB: OuterAllocaBlock, Ident,
2105 IfCondition, NumThreads, PrivTID, PrivTIDAddr,
2106 ThreadID, ToBeDeleted: ToBeDeletedVec);
2107 };
2108 } else {
2109 // Generate OpenMP host runtime call
2110 OI->PostOutlineCB = [=, ToBeDeletedVec =
2111 std::move(ToBeDeleted)](Function &OutlinedFn) {
2112 hostParallelCallback(OMPIRBuilder: this, OutlinedFn, OuterFn, Ident, IfCondition,
2113 PrivTID, PrivTIDAddr, ToBeDeleted: ToBeDeletedVec);
2114 };
2115 }
2116
2117 OI->FixUpNonEntryAllocas = true;
2118 OI->OuterAllocBB = OuterAllocaBlock;
2119 OI->EntryBB = PRegEntryBB;
2120 OI->ExitBB = PRegExitBB;
2121 OI->OuterDeallocBBs.reserve(N: OuterDeallocBlocks.size());
2122 copy(Range&: OuterDeallocBlocks, Out: OI->OuterDeallocBBs.end());
2123
2124 SmallPtrSet<BasicBlock *, 32> ParallelRegionBlockSet;
2125 SmallVector<BasicBlock *, 32> Blocks;
2126 OI->collectBlocks(BlockSet&: ParallelRegionBlockSet, BlockVector&: Blocks);
2127
2128 CodeExtractorAnalysisCache CEAC(*OuterFn);
2129 CodeExtractor Extractor(Blocks, /* DominatorTree */ nullptr,
2130 /* AggregateArgs */ false,
2131 /* BlockFrequencyInfo */ nullptr,
2132 /* BranchProbabilityInfo */ nullptr,
2133 /* AssumptionCache */ nullptr,
2134 /* AllowVarArgs */ true,
2135 /* AllowAlloca */ true,
2136 /* AllocationBlock */ OuterAllocaBlock,
2137 /* DeallocationBlocks */ {},
2138 /* Suffix */ ".omp_par", ArgsInZeroAddressSpace);
2139
2140 // Find inputs to, outputs from the code region.
2141 BasicBlock *CommonExit = nullptr;
2142 SetVector<Value *> Inputs, Outputs, SinkingCands, HoistingCands;
2143 Extractor.findAllocas(CEAC, SinkCands&: SinkingCands, HoistCands&: HoistingCands, ExitBlock&: CommonExit);
2144
2145 Extractor.findInputsOutputs(Inputs, Outputs, Allocas: SinkingCands,
2146 /*CollectGlobalInputs=*/true);
2147
2148 Inputs.remove_if(P: [&](Value *I) {
2149 if (auto *GV = dyn_cast_if_present<GlobalVariable>(Val: I))
2150 return GV->getValueType() == OpenMPIRBuilder::Ident;
2151
2152 return false;
2153 });
2154
2155 LLVM_DEBUG(dbgs() << "Before privatization: " << *OuterFn << "\n");
2156
2157 FunctionCallee TIDRTLFn =
2158 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_global_thread_num);
2159
2160 auto PrivHelper = [&](Value &V) -> Error {
2161 if (&V == TIDAddr || &V == ZeroAddr) {
2162 OI->ExcludeArgsFromAggregate.push_back(Elt: &V);
2163 return Error::success();
2164 }
2165
2166 SetVector<Use *> Uses;
2167 for (Use &U : V.uses())
2168 if (auto *UserI = dyn_cast<Instruction>(Val: U.getUser()))
2169 if (ParallelRegionBlockSet.count(Ptr: UserI->getParent()))
2170 Uses.insert(X: &U);
2171
2172 // __kmpc_fork_call expects extra arguments as pointers. If the input
2173 // already has a pointer type, everything is fine. Otherwise, store the
2174 // value onto stack and load it back inside the to-be-outlined region. This
2175 // will ensure only the pointer will be passed to the function.
2176 // FIXME: if there are more than 15 trailing arguments, they must be
2177 // additionally packed in a struct.
2178 Value *Inner = &V;
2179 if (!V.getType()->isPointerTy()) {
2180 IRBuilder<>::InsertPointGuard Guard(Builder);
2181 LLVM_DEBUG(llvm::dbgs() << "Forwarding input as pointer: " << V << "\n");
2182
2183 Builder.restoreIP(IP: OuterAllocIP);
2184 Value *Ptr;
2185 if (UsesDeviceSharedMemory) {
2186 // Use device shared memory instead, if needed.
2187 Ptr = createOMPAllocShared(Loc: Builder, VarType: V.getType(),
2188 Name: V.getName() + ".reloaded");
2189 for (BasicBlock *DeallocBlock : OuterDeallocBlocks) {
2190 assert(DeallocBlock->getParent() ==
2191 OuterAllocIP.getBlock()->getParent() &&
2192 "Dealloc block must be in the allocation's function to reuse "
2193 "its debug location");
2194 createOMPFreeShared(
2195 Loc: {InsertPointTy(DeallocBlock, DeallocBlock->getFirstInsertionPt()),
2196 Builder.getCurrentDebugLocation()},
2197 Addr: Ptr, VarType: V.getType());
2198 }
2199 } else {
2200 Ptr = Builder.CreateAlloca(Ty: V.getType(), ArraySize: nullptr,
2201 Name: V.getName() + ".reloaded");
2202 }
2203
2204 // Store to stack at end of the block that currently branches to the entry
2205 // block of the to-be-outlined region.
2206 Builder.SetInsertPoint(TheBB: InsertBB,
2207 IP: InsertBB->getTerminator()->getIterator());
2208 Builder.CreateStore(Val: &V, Ptr);
2209
2210 // Load back next to allocations in the to-be-outlined region.
2211 Builder.restoreIP(IP: InnerAllocaIP);
2212 Inner = Builder.CreateLoad(Ty: V.getType(), Ptr);
2213 }
2214
2215 Value *ReplacementValue = nullptr;
2216 CallInst *CI = dyn_cast<CallInst>(Val: &V);
2217 if (CI && CI->getCalledFunction() == TIDRTLFn.getCallee()) {
2218 ReplacementValue = PrivTID;
2219 } else {
2220 InsertPointOrErrorTy AfterIP =
2221 PrivCB(InnerAllocaIP, Builder.saveIP(), V, *Inner, ReplacementValue);
2222 if (!AfterIP)
2223 return AfterIP.takeError();
2224 Builder.restoreIP(IP: *AfterIP);
2225 InnerAllocaIP = {
2226 InnerAllocaIP.getBlock(),
2227 InnerAllocaIP.getBlock()->getTerminator()->getIterator()};
2228
2229 assert(ReplacementValue &&
2230 "Expected copy/create callback to set replacement value!");
2231 if (ReplacementValue == &V)
2232 return Error::success();
2233 }
2234
2235 for (Use *UPtr : Uses)
2236 UPtr->set(ReplacementValue);
2237
2238 return Error::success();
2239 };
2240
2241 // Reset the inner alloca insertion as it will be used for loading the values
2242 // wrapped into pointers before passing them into the to-be-outlined region.
2243 // Configure it to insert immediately after the fake use of zero address so
2244 // that they are available in the generated body and so that the
2245 // OpenMP-related values (thread ID and zero address pointers) remain leading
2246 // in the argument list.
2247 InnerAllocaIP = IRBuilder<>::InsertPoint(
2248 ZeroAddrUse->getParent(), ZeroAddrUse->getNextNode()->getIterator());
2249
2250 // Reset the outer alloca insertion point to the entry of the relevant block
2251 // in case it was invalidated.
2252 OuterAllocIP = IRBuilder<>::InsertPoint(
2253 OuterAllocaBlock, OuterAllocaBlock->getFirstInsertionPt());
2254
2255 for (Value *Input : Inputs) {
2256 LLVM_DEBUG(dbgs() << "Captured input: " << *Input << "\n");
2257 if (Error Err = PrivHelper(*Input))
2258 return Err;
2259 }
2260 LLVM_DEBUG({
2261 for (Value *Output : Outputs)
2262 LLVM_DEBUG(dbgs() << "Captured output: " << *Output << "\n");
2263 });
2264 assert(Outputs.empty() &&
2265 "OpenMP outlining should not produce live-out values!");
2266
2267 LLVM_DEBUG(dbgs() << "After privatization: " << *OuterFn << "\n");
2268 LLVM_DEBUG({
2269 for (auto *BB : Blocks)
2270 dbgs() << " PBR: " << BB->getName() << "\n";
2271 });
2272
2273 // Adjust the finalization stack, verify the adjustment, and call the
2274 // finalize function a last time to finalize values between the pre-fini
2275 // block and the exit block if we left the parallel "the normal way".
2276 auto FiniInfo = FinalizationStack.pop_back_val();
2277 (void)FiniInfo;
2278 assert(FiniInfo.DK == OMPD_parallel &&
2279 "Unexpected finalization stack state!");
2280
2281 Instruction *PRegPreFiniTI = PRegPreFiniBB->getTerminator();
2282
2283 InsertPointTy PreFiniIP(PRegPreFiniBB, PRegPreFiniTI->getIterator());
2284 Expected<BasicBlock *> FiniBBOrErr = FiniInfo.getFiniBB(Builder);
2285 if (!FiniBBOrErr)
2286 return FiniBBOrErr.takeError();
2287 {
2288 IRBuilderBase::InsertPointGuard Guard(Builder);
2289 Builder.restoreIP(IP: PreFiniIP);
2290 Builder.CreateBr(Dest: *FiniBBOrErr);
2291 // There's currently a branch to omp.par.exit. Delete it. We will get there
2292 // via the fini block
2293 if (Instruction *Term = Builder.GetInsertBlock()->getTerminator())
2294 Term->eraseFromParent();
2295 }
2296
2297 // Register the outlined info.
2298 addOutlineInfo(OI: std::move(OI));
2299
2300 InsertPointTy AfterIP(UI->getParent(), UI->getParent()->end());
2301 UI->eraseFromParent();
2302
2303 return AfterIP;
2304}
2305
2306void OpenMPIRBuilder::emitFlush(const LocationDescription &Loc) {
2307 // Build call void __kmpc_flush(ident_t *loc)
2308 uint32_t SrcLocStrSize;
2309 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2310 Value *Args[] = {getOrCreateIdent(SrcLocStr, SrcLocStrSize)};
2311
2312 createRuntimeFunctionCall(Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_flush),
2313 Args);
2314}
2315
2316void OpenMPIRBuilder::createFlush(const LocationDescription &Loc) {
2317 if (!updateToLocation(Loc))
2318 return;
2319 emitFlush(Loc);
2320}
2321
2322void OpenMPIRBuilder::createError(const LocationDescription &Loc, bool IsFatal,
2323 Value *Message) {
2324 if (!updateToLocation(Loc))
2325 return;
2326
2327 // Build call void __kmpc_error(ident_t *loc, int severity,
2328 // const char *message)
2329 uint32_t SrcLocStrSize;
2330 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2331 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2332 // Severity: 1 = warning, 2 = fatal.
2333 Value *Severity = ConstantInt::get(Ty: Int32, V: IsFatal ? 2 : 1);
2334 Value *MessageArg = Message ? Message : ConstantPointerNull::get(T: Int8Ptr);
2335 Value *Args[] = {Ident, Severity, MessageArg};
2336
2337 createRuntimeFunctionCall(Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_error),
2338 Args);
2339}
2340
2341void OpenMPIRBuilder::emitTaskyieldImpl(const LocationDescription &Loc) {
2342 // Build call __kmpc_omp_taskyield(loc, thread_id, 0);
2343 uint32_t SrcLocStrSize;
2344 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2345 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2346 Constant *I32Null = ConstantInt::getNullValue(Ty: Int32);
2347 Value *Args[] = {Ident, getOrCreateThreadID(Ident), I32Null};
2348
2349 createRuntimeFunctionCall(
2350 Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_taskyield), Args);
2351}
2352
2353void OpenMPIRBuilder::createTaskyield(const LocationDescription &Loc) {
2354 if (!updateToLocation(Loc))
2355 return;
2356 emitTaskyieldImpl(Loc);
2357}
2358
2359void OpenMPIRBuilder::emitTaskDependency(IRBuilderBase &Builder, Value *Entry,
2360 const DependData &Dep) {
2361 // Store the pointer to the variable
2362 Value *Addr = Builder.CreateStructGEP(
2363 Ty: DependInfo, Ptr: Entry,
2364 Idx: static_cast<unsigned int>(RTLDependInfoFields::BaseAddr));
2365 Value *DepValPtr = Builder.CreatePtrToInt(V: Dep.DepVal, DestTy: SizeTy);
2366 Builder.CreateStore(Val: DepValPtr, Ptr: Addr);
2367 // Store the size of the variable
2368 Value *Size = Builder.CreateStructGEP(
2369 Ty: DependInfo, Ptr: Entry, Idx: static_cast<unsigned int>(RTLDependInfoFields::Len));
2370 Builder.CreateStore(
2371 Val: ConstantInt::get(Ty: SizeTy,
2372 V: M.getDataLayout().getTypeStoreSize(Ty: Dep.DepValueType)),
2373 Ptr: Size);
2374 // Store the dependency kind
2375 Value *Flags = Builder.CreateStructGEP(
2376 Ty: DependInfo, Ptr: Entry, Idx: static_cast<unsigned int>(RTLDependInfoFields::Flags));
2377 Builder.CreateStore(Val: ConstantInt::get(Ty: Builder.getInt8Ty(),
2378 V: static_cast<unsigned int>(Dep.DepKind)),
2379 Ptr: Flags);
2380}
2381
2382// Processes the dependencies in Dependencies and does the following
2383// - Allocates space on the stack of an array of DependInfo objects
2384// - Populates each DependInfo object with relevant information of
2385// the corresponding dependence.
2386// - All code is inserted in the entry block of the current function.
2387static Value *emitTaskDependencies(
2388 OpenMPIRBuilder &OMPBuilder,
2389 const SmallVectorImpl<OpenMPIRBuilder::DependData> &Dependencies) {
2390 // Early return if we have no dependencies to process
2391 if (Dependencies.empty())
2392 return nullptr;
2393
2394 // Given a vector of DependData objects, in this function we create an
2395 // array on the stack that holds kmp_depend_info objects corresponding
2396 // to each dependency. This is then passed to the OpenMP runtime.
2397 // For example, if there are 'n' dependencies then the following psedo
2398 // code is generated. Assume the first dependence is on a variable 'a'
2399 //
2400 // \code{c}
2401 // DepArray = alloc(n x sizeof(kmp_depend_info);
2402 // idx = 0;
2403 // DepArray[idx].base_addr = ptrtoint(&a);
2404 // DepArray[idx].len = 8;
2405 // DepArray[idx].flags = Dep.DepKind; /*(See OMPContants.h for DepKind)*/
2406 // ++idx;
2407 // DepArray[idx].base_addr = ...;
2408 // \endcode
2409
2410 IRBuilderBase &Builder = OMPBuilder.Builder;
2411 Type *DependInfo = OMPBuilder.DependInfo;
2412
2413 Value *DepArray = nullptr;
2414 Type *DepArrayTy = ArrayType::get(ElementType: DependInfo, NumElements: Dependencies.size());
2415 {
2416 // Use a InsertPointGuard to restore the location back along with the
2417 // insertion point.
2418 IRBuilderBase::InsertPointGuard IPGuard(Builder);
2419 Builder.SetInsertPoint(
2420 Builder.GetInsertBlock()->getParent()->getEntryBlock().getTerminator());
2421 DepArray = Builder.CreateAlloca(Ty: DepArrayTy, ArraySize: nullptr, Name: ".dep.arr.addr");
2422 }
2423
2424 for (const auto &[DepIdx, Dep] : enumerate(First: Dependencies)) {
2425 Value *Base =
2426 Builder.CreateConstInBoundsGEP2_64(Ty: DepArrayTy, Ptr: DepArray, Idx0: 0, Idx1: DepIdx);
2427 OMPBuilder.emitTaskDependency(Builder, Entry: Base, Dep);
2428 }
2429 return DepArray;
2430}
2431
2432void OpenMPIRBuilder::emitTaskwaitImpl(const LocationDescription &Loc) {
2433 // Build call kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
2434 // global_tid);
2435 uint32_t SrcLocStrSize;
2436 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2437 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2438 Value *Args[] = {Ident, getOrCreateThreadID(Ident)};
2439
2440 // Ignore return result until untied tasks are supported.
2441 createRuntimeFunctionCall(
2442 Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_taskwait), Args);
2443}
2444
2445void OpenMPIRBuilder::createTaskwait(const LocationDescription &Loc,
2446 DependenciesInfo Dependencies) {
2447 if (!updateToLocation(Loc))
2448 return;
2449
2450 Value *DepArray = nullptr;
2451 Type *DepArrayTy = nullptr;
2452 Value *NumDeps = nullptr;
2453 if (Dependencies.DepArray) {
2454 DepArray = Dependencies.DepArray;
2455 NumDeps = Dependencies.NumDeps;
2456 } else if (!Dependencies.Deps.empty()) {
2457 DepArrayTy = ArrayType::get(ElementType: DependInfo, NumElements: Dependencies.Deps.size());
2458 NumDeps = Builder.getInt32(C: Dependencies.Deps.size());
2459 {
2460 IRBuilderBase::InsertPointGuard IPGuard(Builder);
2461 BasicBlock &entryBB =
2462 Builder.GetInsertBlock()->getParent()->getEntryBlock();
2463 Builder.SetInsertPoint(TheBB: &entryBB, IP: entryBB.getFirstInsertionPt());
2464 DepArray = Builder.CreateAlloca(Ty: DepArrayTy, ArraySize: nullptr, Name: ".dep.arr.addr");
2465 }
2466
2467 for (const auto &[DepIdx, Dep] : enumerate(First&: Dependencies.Deps)) {
2468 Value *Base =
2469 Builder.CreateConstInBoundsGEP2_64(Ty: DepArrayTy, Ptr: DepArray, Idx0: 0, Idx1: DepIdx);
2470 this->emitTaskDependency(Builder, Entry: Base, Dep);
2471 }
2472 }
2473
2474 if (DepArray) {
2475 uint32_t SrcLocStrSize;
2476 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2477 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2478 Value *Args[] = {
2479 Ident,
2480 getOrCreateThreadID(Ident),
2481 NumDeps,
2482 DepArray,
2483 ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0),
2484 ConstantPointerNull::get(T: PointerType::getUnqual(C&: M.getContext())),
2485 ConstantInt::get(Ty: Builder.getInt32Ty(), V: false)};
2486 createRuntimeFunctionCall(
2487 Callee: getOrCreateRuntimeFunctionPtr(
2488 FnID: omp::RuntimeFunction::OMPRTL___kmpc_omp_taskwait_deps_51),
2489 Args);
2490 } else {
2491 emitTaskwaitImpl(Loc);
2492 }
2493}
2494
2495/// Create the task duplication function passed to kmpc_taskloop.
2496Expected<Value *> OpenMPIRBuilder::createTaskDuplicationFunction(
2497 Type *PrivatesTy, int32_t PrivatesIndex, TaskDupCallbackTy DupCB) {
2498 unsigned ProgramAddressSpace = M.getDataLayout().getProgramAddressSpace();
2499 if (!DupCB)
2500 return Constant::getNullValue(
2501 Ty: PointerType::get(C&: Builder.getContext(), AddressSpace: ProgramAddressSpace));
2502
2503 // From OpenMP Runtime p_task_dup_t:
2504 // Routine optionally generated by the compiler for setting the lastprivate
2505 // flag and calling needed constructors for private/firstprivate objects (used
2506 // to form taskloop tasks from pattern task) Parameters: dest task, src task,
2507 // lastprivate flag.
2508 // typedef void (*p_task_dup_t)(kmp_task_t *, kmp_task_t *, kmp_int32);
2509
2510 auto *VoidPtrTy = PointerType::get(C&: Builder.getContext(), AddressSpace: ProgramAddressSpace);
2511
2512 FunctionType *DupFuncTy = FunctionType::get(
2513 Result: Builder.getVoidTy(), Params: {VoidPtrTy, VoidPtrTy, Builder.getInt32Ty()},
2514 /*isVarArg=*/false);
2515
2516 Function *DupFunction = Function::Create(Ty: DupFuncTy, Linkage: Function::InternalLinkage,
2517 N: "omp_taskloop_dup", M);
2518 Value *DestTaskArg = DupFunction->getArg(i: 0);
2519 Value *SrcTaskArg = DupFunction->getArg(i: 1);
2520 Value *LastprivateFlagArg = DupFunction->getArg(i: 2);
2521 DestTaskArg->setName("dest_task");
2522 SrcTaskArg->setName("src_task");
2523 LastprivateFlagArg->setName("lastprivate_flag");
2524
2525 IRBuilderBase::InsertPointGuard Guard(Builder);
2526 Builder.SetInsertPoint(
2527 BasicBlock::Create(Context&: Builder.getContext(), Name: "entry", Parent: DupFunction));
2528
2529 auto GetTaskContextPtrFromArg = [&](Value *Arg) -> Value * {
2530 Type *TaskWithPrivatesTy =
2531 StructType::get(Context&: Builder.getContext(), Elements: {Task, PrivatesTy});
2532 Value *TaskPrivates = Builder.CreateGEP(
2533 Ty: TaskWithPrivatesTy, Ptr: Arg, IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: 1)});
2534 Value *ContextPtr = Builder.CreateGEP(
2535 Ty: PrivatesTy, Ptr: TaskPrivates,
2536 IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: PrivatesIndex)});
2537 return ContextPtr;
2538 };
2539
2540 Value *DestTaskContextPtr = GetTaskContextPtrFromArg(DestTaskArg);
2541 Value *SrcTaskContextPtr = GetTaskContextPtrFromArg(SrcTaskArg);
2542
2543 DestTaskContextPtr->setName("destPtr");
2544 SrcTaskContextPtr->setName("srcPtr");
2545
2546 InsertPointTy AllocaIP(&DupFunction->getEntryBlock(),
2547 DupFunction->getEntryBlock().begin());
2548 InsertPointTy CodeGenIP = Builder.saveIP();
2549 Expected<IRBuilderBase::InsertPoint> AfterIPOrError =
2550 DupCB(AllocaIP, CodeGenIP, DestTaskContextPtr, SrcTaskContextPtr);
2551 if (!AfterIPOrError)
2552 return AfterIPOrError.takeError();
2553 Builder.restoreIP(IP: *AfterIPOrError);
2554
2555 Builder.CreateRetVoid();
2556
2557 return DupFunction;
2558}
2559
2560OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTaskloop(
2561 const LocationDescription &Loc, InsertPointTy AllocaIP,
2562 ArrayRef<BasicBlock *> DeallocBlocks, BodyGenCallbackTy BodyGenCB,
2563 llvm::function_ref<llvm::Expected<llvm::CanonicalLoopInfo *>()> LoopInfo,
2564 Value *LBVal, Value *UBVal, Value *StepVal, bool Untied, Value *IfCond,
2565 Value *GrainSize, bool NoGroup, int Sched, Value *Final, bool Mergeable,
2566 Value *Priority, uint64_t NumOfCollapseLoops, TaskDupCallbackTy DupCB,
2567 Value *TaskContextStructPtrVal, bool FreeAgent) {
2568
2569 if (!updateToLocation(Loc))
2570 return InsertPointTy();
2571
2572 uint32_t SrcLocStrSize;
2573 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2574 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2575
2576 BasicBlock *TaskloopExitBB =
2577 splitBB(Builder, /*CreateBranch=*/true, Name: "taskloop.exit");
2578 BasicBlock *TaskloopBodyBB =
2579 splitBB(Builder, /*CreateBranch=*/true, Name: "taskloop.body");
2580 BasicBlock *TaskloopAllocaBB =
2581 splitBB(Builder, /*CreateBranch=*/true, Name: "taskloop.alloca");
2582
2583 InsertPointTy TaskloopAllocaIP =
2584 InsertPointTy(TaskloopAllocaBB, TaskloopAllocaBB->begin());
2585 InsertPointTy TaskloopBodyIP =
2586 InsertPointTy(TaskloopBodyBB, TaskloopBodyBB->begin());
2587
2588 if (Error Err = BodyGenCB(TaskloopAllocaIP, TaskloopBodyIP, TaskloopExitBB))
2589 return Err;
2590
2591 llvm::Expected<llvm::CanonicalLoopInfo *> result = LoopInfo();
2592 if (!result) {
2593 return result.takeError();
2594 }
2595
2596 llvm::CanonicalLoopInfo *CLI = result.get();
2597 auto OI = std::make_unique<OutlineInfo>();
2598 OI->EntryBB = TaskloopAllocaBB;
2599 OI->OuterAllocBB = AllocaIP.getBlock();
2600 OI->ExitBB = TaskloopExitBB;
2601 OI->OuterDeallocBBs.reserve(N: DeallocBlocks.size());
2602 copy(Range&: DeallocBlocks, Out: OI->OuterDeallocBBs.end());
2603
2604 // Add the thread ID argument.
2605 SmallVector<Instruction *> ToBeDeleted;
2606 // dummy instruction to be used as a fake argument
2607 OI->ExcludeArgsFromAggregate.push_back(Elt: createFakeIntVal(
2608 Builder, OuterAllocaIP: AllocaIP, ToBeDeleted, InnerAllocaIP: TaskloopAllocaIP, Name: "global.tid", AsPtr: false));
2609 Value *FakeLB = createFakeIntVal(Builder, OuterAllocaIP: AllocaIP, ToBeDeleted,
2610 InnerAllocaIP: TaskloopAllocaIP, Name: "lb", AsPtr: false, Is64Bit: true);
2611 Value *FakeUB = createFakeIntVal(Builder, OuterAllocaIP: AllocaIP, ToBeDeleted,
2612 InnerAllocaIP: TaskloopAllocaIP, Name: "ub", AsPtr: false, Is64Bit: true);
2613 Value *FakeStep = createFakeIntVal(Builder, OuterAllocaIP: AllocaIP, ToBeDeleted,
2614 InnerAllocaIP: TaskloopAllocaIP, Name: "step", AsPtr: false, Is64Bit: true);
2615 // For Taskloop, we want to force the bounds being the first 3 inputs in the
2616 // aggregate struct
2617 OI->Inputs.insert(X: FakeLB);
2618 OI->Inputs.insert(X: FakeUB);
2619 OI->Inputs.insert(X: FakeStep);
2620 if (TaskContextStructPtrVal)
2621 OI->Inputs.insert(X: TaskContextStructPtrVal);
2622 assert(((TaskContextStructPtrVal && DupCB) ||
2623 (!TaskContextStructPtrVal && !DupCB)) &&
2624 "Task context struct ptr and duplication callback must be both set "
2625 "or both null");
2626
2627 // It isn't safe to run the duplication bodygen callback inside the post
2628 // outlining callback so this has to be run now before we know the real task
2629 // shareds structure type.
2630 unsigned ProgramAddressSpace = M.getDataLayout().getProgramAddressSpace();
2631 Type *PointerTy = PointerType::get(C&: Builder.getContext(), AddressSpace: ProgramAddressSpace);
2632 Type *FakeSharedsTy = StructType::get(
2633 Context&: Builder.getContext(),
2634 Elements: {FakeLB->getType(), FakeUB->getType(), FakeStep->getType(), PointerTy});
2635 Expected<Value *> TaskDupFnOrErr = createTaskDuplicationFunction(
2636 PrivatesTy: FakeSharedsTy,
2637 /*PrivatesIndex: the pointer after the three indices above*/ PrivatesIndex: 3, DupCB);
2638 if (!TaskDupFnOrErr) {
2639 return TaskDupFnOrErr.takeError();
2640 }
2641 Value *TaskDupFn = *TaskDupFnOrErr;
2642
2643 OI->PostOutlineCB = [this, Ident, LBVal, UBVal, StepVal, Untied,
2644 TaskloopAllocaBB, CLI, TaskDupFn, ToBeDeleted, IfCond,
2645 GrainSize, NoGroup, Sched, FakeLB, FakeUB, FakeStep,
2646 FakeSharedsTy, Final, Mergeable, Priority,
2647 NumOfCollapseLoops,
2648 FreeAgent](Function &OutlinedFn) mutable {
2649 // Replace the Stale CI by appropriate RTL function call.
2650 assert(OutlinedFn.hasOneUse() &&
2651 "there must be a single user for the outlined function");
2652 CallInst *StaleCI = cast<CallInst>(Val: OutlinedFn.user_back());
2653
2654 /* Create the casting for the Bounds Values that can be used when outlining
2655 * to replace the uses of the fakes with real values */
2656 BasicBlock *CodeReplBB = StaleCI->getParent();
2657 Builder.SetInsertPoint(CodeReplBB->getFirstInsertionPt());
2658 Value *CastedLBVal =
2659 Builder.CreateIntCast(V: LBVal, DestTy: Builder.getInt64Ty(), isSigned: true, Name: "lb64");
2660 Value *CastedUBVal =
2661 Builder.CreateIntCast(V: UBVal, DestTy: Builder.getInt64Ty(), isSigned: true, Name: "ub64");
2662 Value *CastedStepVal =
2663 Builder.CreateIntCast(V: StepVal, DestTy: Builder.getInt64Ty(), isSigned: true, Name: "step64");
2664
2665 Builder.SetInsertPoint(StaleCI);
2666
2667 // Gather the arguments for emitting the runtime call for
2668 // @__kmpc_omp_task_alloc
2669 Function *TaskAllocFn =
2670 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_alloc);
2671
2672 Value *ThreadID = getOrCreateThreadID(Ident);
2673
2674 if (!NoGroup) {
2675 // Emit runtime call for @__kmpc_taskgroup
2676 Function *TaskgroupFn =
2677 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_taskgroup);
2678 Builder.CreateCall(Callee: TaskgroupFn, Args: {Ident, ThreadID});
2679 }
2680
2681 // `flags` Argument Configuration
2682 // Task is tied if (Flags & 1) == 1.
2683 // Task is untied if (Flags & 1) == 0.
2684 // Task is final if (Flags & 2) == 2.
2685 // Task is not final if (Flags & 2) == 0.
2686 // Task is mergeable if (Flags & 4) == 4.
2687 // Task is not mergeable if (Flags & 4) == 0.
2688 // Task is priority if (Flags & 32) == 32.
2689 // Task is not priority if (Flags & 32) == 0.
2690 // Task is free-agent eligible if (Flags & 128) == 128.
2691 // Task is not free-agent eligible if (Flags & 128) == 0.
2692 Value *Flags = Builder.getInt32(C: Untied ? 0 : 1);
2693 if (Final)
2694 Flags = Builder.CreateOr(LHS: Builder.getInt32(C: 2), RHS: Flags);
2695 if (Mergeable)
2696 Flags = Builder.CreateOr(LHS: Builder.getInt32(C: 4), RHS: Flags);
2697 if (Priority)
2698 Flags = Builder.CreateOr(LHS: Builder.getInt32(C: 32), RHS: Flags);
2699 if (FreeAgent)
2700 Flags = Builder.CreateOr(LHS: Builder.getInt32(C: 128), RHS: Flags);
2701
2702 Value *TaskSize = Builder.getInt64(
2703 C: divideCeil(Numerator: M.getDataLayout().getTypeSizeInBits(Ty: Task), Denominator: 8));
2704
2705 AllocaInst *ArgStructAlloca =
2706 dyn_cast<AllocaInst>(Val: StaleCI->getArgOperand(i: 1));
2707 assert(ArgStructAlloca &&
2708 "Unable to find the alloca instruction corresponding to arguments "
2709 "for extracted function");
2710 std::optional<TypeSize> ArgAllocSize =
2711 ArgStructAlloca->getAllocationSize(DL: M.getDataLayout());
2712 assert(ArgAllocSize &&
2713 "Unable to determine size of arguments for extracted function");
2714 Value *SharedsSize = Builder.getInt64(C: ArgAllocSize->getFixedValue());
2715
2716 // Emit the @__kmpc_omp_task_alloc runtime call
2717 // The runtime call returns a pointer to an area where the task captured
2718 // variables must be copied before the task is run (TaskData)
2719 CallInst *TaskData = Builder.CreateCall(
2720 Callee: TaskAllocFn, Args: {/*loc_ref=*/Ident, /*gtid=*/ThreadID, /*flags=*/Flags,
2721 /*sizeof_task=*/TaskSize, /*sizeof_shared=*/SharedsSize,
2722 /*task_func=*/&OutlinedFn});
2723
2724 Value *Shareds = StaleCI->getArgOperand(i: 1);
2725 Align Alignment = TaskData->getPointerAlignment(DL: M.getDataLayout());
2726 Value *TaskShareds = Builder.CreateLoad(Ty: VoidPtr, Ptr: TaskData);
2727 Builder.CreateMemCpy(Dst: TaskShareds, DstAlign: Alignment, Src: Shareds, SrcAlign: Alignment,
2728 Size: SharedsSize);
2729 // Get the pointer to loop lb, ub, step from task ptr
2730 // and set up the lowerbound,upperbound and step values
2731 llvm::Value *Lb = Builder.CreateGEP(
2732 Ty: FakeSharedsTy, Ptr: TaskShareds, IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: 0)});
2733
2734 llvm::Value *Ub = Builder.CreateGEP(
2735 Ty: FakeSharedsTy, Ptr: TaskShareds, IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: 1)});
2736
2737 llvm::Value *Step = Builder.CreateGEP(
2738 Ty: FakeSharedsTy, Ptr: TaskShareds, IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: 2)});
2739 llvm::Value *Loadstep = Builder.CreateLoad(Ty: Builder.getInt64Ty(), Ptr: Step);
2740
2741 // set up the arguments for emitting kmpc_taskloop runtime call
2742 // setting values for ifval, nogroup, sched, grainsize, task_dup
2743 Value *IfCondVal =
2744 IfCond ? Builder.CreateIntCast(V: IfCond, DestTy: Builder.getInt32Ty(), isSigned: true)
2745 : Builder.getInt32(C: 1);
2746 // As __kmpc_taskgroup is called manually in OMPIRBuilder, NoGroupVal should
2747 // always be 1 when calling __kmpc_taskloop to ensure it is not called again
2748 Value *NoGroupVal = Builder.getInt32(C: 1);
2749 Value *SchedVal = Builder.getInt32(C: Sched);
2750 Value *GrainSizeVal =
2751 GrainSize ? Builder.CreateIntCast(V: GrainSize, DestTy: Builder.getInt64Ty(), isSigned: true)
2752 : Builder.getInt64(C: 0);
2753 Value *TaskDup = TaskDupFn;
2754
2755 Value *Args[] = {Ident, ThreadID, TaskData, IfCondVal, Lb, Ub,
2756 Loadstep, NoGroupVal, SchedVal, GrainSizeVal, TaskDup};
2757
2758 // taskloop runtime call
2759 Function *TaskloopFn =
2760 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_taskloop);
2761 Builder.CreateCall(Callee: TaskloopFn, Args);
2762
2763 // Emit the @__kmpc_end_taskgroup runtime call to end the taskgroup if
2764 // nogroup is not defined
2765 if (!NoGroup) {
2766 Function *EndTaskgroupFn =
2767 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_end_taskgroup);
2768 Builder.CreateCall(Callee: EndTaskgroupFn, Args: {Ident, ThreadID});
2769 }
2770
2771 StaleCI->eraseFromParent();
2772
2773 Builder.SetInsertPoint(TheBB: TaskloopAllocaBB, IP: TaskloopAllocaBB->begin());
2774
2775 LoadInst *SharedsOutlined =
2776 Builder.CreateLoad(Ty: VoidPtr, Ptr: OutlinedFn.getArg(i: 1));
2777 OutlinedFn.getArg(i: 1)->replaceUsesWithIf(
2778 New: SharedsOutlined,
2779 ShouldReplace: [SharedsOutlined](Use &U) { return U.getUser() != SharedsOutlined; });
2780
2781 Value *IV = CLI->getIndVar();
2782 Type *IVTy = IV->getType();
2783 Constant *One = ConstantInt::get(Ty: Builder.getInt64Ty(), V: 1);
2784
2785 // When outlining, CodeExtractor will create GEP's to the LowerBound and
2786 // UpperBound. These GEP's can be reused for loading the tasks respective
2787 // bounds.
2788 Value *TaskLB = nullptr;
2789 Value *TaskUB = nullptr;
2790 Value *TaskStep = nullptr;
2791 Value *LoadTaskLB = nullptr;
2792 Value *LoadTaskUB = nullptr;
2793 Value *LoadTaskStep = nullptr;
2794 for (Instruction &I : *TaskloopAllocaBB) {
2795 if (I.getOpcode() == Instruction::GetElementPtr) {
2796 GetElementPtrInst &Gep = cast<GetElementPtrInst>(Val&: I);
2797 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: Gep.getOperand(i_nocapture: 2))) {
2798 switch (CI->getZExtValue()) {
2799 case 0:
2800 TaskLB = &I;
2801 break;
2802 case 1:
2803 TaskUB = &I;
2804 break;
2805 case 2:
2806 TaskStep = &I;
2807 break;
2808 }
2809 }
2810 } else if (I.getOpcode() == Instruction::Load) {
2811 LoadInst &Load = cast<LoadInst>(Val&: I);
2812 if (Load.getPointerOperand() == TaskLB) {
2813 assert(TaskLB != nullptr && "Expected value for TaskLB");
2814 LoadTaskLB = &I;
2815 } else if (Load.getPointerOperand() == TaskUB) {
2816 assert(TaskUB != nullptr && "Expected value for TaskUB");
2817 LoadTaskUB = &I;
2818 } else if (Load.getPointerOperand() == TaskStep) {
2819 assert(TaskStep != nullptr && "Expected value for TaskStep");
2820 LoadTaskStep = &I;
2821 }
2822 }
2823 }
2824
2825 Builder.SetInsertPoint(CLI->getPreheader()->getTerminator());
2826
2827 assert(LoadTaskLB != nullptr && "Expected value for LoadTaskLB");
2828 assert(LoadTaskUB != nullptr && "Expected value for LoadTaskUB");
2829 assert(LoadTaskStep != nullptr && "Expected value for LoadTaskStep");
2830 Value *TripCountMinusOne = Builder.CreateSDiv(
2831 LHS: Builder.CreateSub(LHS: LoadTaskUB, RHS: LoadTaskLB), RHS: LoadTaskStep);
2832 Value *TripCount = Builder.CreateAdd(LHS: TripCountMinusOne, RHS: One, Name: "trip_cnt");
2833 Value *CastedTripCount = Builder.CreateIntCast(V: TripCount, DestTy: IVTy, isSigned: true);
2834 Value *CastedTaskLB = Builder.CreateIntCast(V: LoadTaskLB, DestTy: IVTy, isSigned: true);
2835 // set the trip count in the CLI
2836 CLI->setTripCount(CastedTripCount);
2837
2838 Builder.SetInsertPoint(TheBB: CLI->getBody(),
2839 IP: CLI->getBody()->getFirstInsertionPt());
2840
2841 if (NumOfCollapseLoops > 1) {
2842 llvm::SmallVector<User *> UsersToReplace;
2843 // When using the collapse clause, the bounds of the loop have to be
2844 // adjusted to properly represent the iterator of the outer loop.
2845 Value *IVPlusTaskLB = Builder.CreateAdd(
2846 LHS: CLI->getIndVar(),
2847 RHS: Builder.CreateSub(LHS: CastedTaskLB, RHS: ConstantInt::get(Ty: IVTy, V: 1)));
2848 // To ensure every Use is correctly captured, we first want to record
2849 // which users to replace the value in, and then replace the value.
2850 for (auto IVUse = CLI->getIndVar()->uses().begin();
2851 IVUse != CLI->getIndVar()->uses().end(); IVUse++) {
2852 User *IVUser = IVUse->getUser();
2853 if (auto *Op = dyn_cast<BinaryOperator>(Val: IVUser)) {
2854 if (Op->getOpcode() == Instruction::URem ||
2855 Op->getOpcode() == Instruction::UDiv) {
2856 UsersToReplace.push_back(Elt: IVUser);
2857 }
2858 }
2859 }
2860 for (User *User : UsersToReplace) {
2861 User->replaceUsesOfWith(From: CLI->getIndVar(), To: IVPlusTaskLB);
2862 }
2863 } else {
2864 // The canonical loop is generated with a fixed lower bound. We need to
2865 // update the index calculation code to use the task's lower bound. The
2866 // generated code looks like this:
2867 // %omp_loop.iv = phi ...
2868 // ...
2869 // %tmp = mul [type] %omp_loop.iv, step
2870 // %user_index = add [type] tmp, lb
2871 // OpenMPIRBuilder constructs canonical loops to have exactly three uses
2872 // of the normalised induction variable:
2873 // 1. This one: converting the normalised IV to the user IV
2874 // 2. The increment (add)
2875 // 3. The comparison against the trip count (icmp)
2876 // (1) is the only use that is a mul followed by an add so this cannot
2877 // match other IR.
2878 assert(CLI->getIndVar()->getNumUses() == 3 &&
2879 "Canonical loop should have exactly three uses of the ind var");
2880 for (User *IVUser : CLI->getIndVar()->users()) {
2881 if (auto *Mul = dyn_cast<BinaryOperator>(Val: IVUser)) {
2882 if (Mul->getOpcode() == Instruction::Mul) {
2883 for (User *MulUser : Mul->users()) {
2884 if (auto *Add = dyn_cast<BinaryOperator>(Val: MulUser)) {
2885 if (Add->getOpcode() == Instruction::Add) {
2886 Add->setOperand(i_nocapture: 1, Val_nocapture: CastedTaskLB);
2887 }
2888 }
2889 }
2890 }
2891 }
2892 }
2893 }
2894
2895 FakeLB->replaceAllUsesWith(V: CastedLBVal);
2896 FakeUB->replaceAllUsesWith(V: CastedUBVal);
2897 FakeStep->replaceAllUsesWith(V: CastedStepVal);
2898 for (Instruction *I : llvm::reverse(C&: ToBeDeleted)) {
2899 I->eraseFromParent();
2900 }
2901 };
2902
2903 addOutlineInfo(OI: std::move(OI));
2904 Builder.SetInsertPoint(TheBB: TaskloopExitBB, IP: TaskloopExitBB->begin());
2905 return Builder.saveIP();
2906}
2907
2908llvm::StructType *OpenMPIRBuilder::getKmpTaskAffinityInfoTy() {
2909 llvm::Type *IntPtrTy = llvm::Type::getIntNTy(
2910 C&: M.getContext(), N: M.getDataLayout().getPointerSizeInBits());
2911 return llvm::StructType::get(elt1: IntPtrTy, elts: IntPtrTy,
2912 elts: llvm::Type::getInt32Ty(C&: M.getContext()));
2913}
2914
2915OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTask(
2916 const LocationDescription &Loc, InsertPointTy AllocaIP,
2917 ArrayRef<BasicBlock *> DeallocBlocks, BodyGenCallbackTy BodyGenCB,
2918 bool Tied, Value *Final, Value *IfCondition,
2919 const DependenciesInfo &Dependencies, const AffinityData &Affinities,
2920 bool Mergeable, Value *EventHandle, Value *Priority, bool FreeAgent) {
2921
2922 if (!updateToLocation(Loc))
2923 return InsertPointTy();
2924
2925 uint32_t SrcLocStrSize;
2926 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2927 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2928 // The current basic block is split into four basic blocks. After outlining,
2929 // they will be mapped as follows:
2930 // ```
2931 // def current_fn() {
2932 // current_basic_block:
2933 // br label %task.exit
2934 // task.exit:
2935 // ; instructions after task
2936 // }
2937 // def outlined_fn() {
2938 // task.alloca:
2939 // br label %task.body
2940 // task.body:
2941 // ret void
2942 // }
2943 // ```
2944 BasicBlock *TaskExitBB = splitBB(Builder, /*CreateBranch=*/true, Name: "task.exit");
2945 BasicBlock *TaskBodyBB = splitBB(Builder, /*CreateBranch=*/true, Name: "task.body");
2946 BasicBlock *TaskAllocaBB =
2947 splitBB(Builder, /*CreateBranch=*/true, Name: "task.alloca");
2948
2949 InsertPointTy TaskAllocaIP =
2950 InsertPointTy(TaskAllocaBB, TaskAllocaBB->begin());
2951 InsertPointTy TaskBodyIP = InsertPointTy(TaskBodyBB, TaskBodyBB->begin());
2952 if (Error Err = BodyGenCB(TaskAllocaIP, TaskBodyIP, TaskExitBB))
2953 return Err;
2954
2955 auto OI = std::make_unique<OutlineInfo>();
2956 OI->EntryBB = TaskAllocaBB;
2957 OI->OuterAllocBB = AllocaIP.getBlock();
2958 OI->ExitBB = TaskExitBB;
2959 OI->OuterDeallocBBs.reserve(N: DeallocBlocks.size());
2960 copy(Range&: DeallocBlocks, Out: OI->OuterDeallocBBs.end());
2961
2962 // Add the thread ID argument.
2963 SmallVector<Instruction *, 4> ToBeDeleted;
2964 OI->ExcludeArgsFromAggregate.push_back(Elt: createFakeIntVal(
2965 Builder, OuterAllocaIP: AllocaIP, ToBeDeleted, InnerAllocaIP: TaskAllocaIP, Name: "global.tid", AsPtr: false));
2966
2967 OI->PostOutlineCB = [this, Ident, Tied, Final, IfCondition, Dependencies,
2968 Affinities, Mergeable, Priority, EventHandle, FreeAgent,
2969 TaskAllocaBB,
2970 ToBeDeleted](Function &OutlinedFn) mutable {
2971 // Replace the Stale CI by appropriate RTL function call.
2972 assert(OutlinedFn.hasOneUse() &&
2973 "there must be a single user for the outlined function");
2974 CallInst *StaleCI = cast<CallInst>(Val: OutlinedFn.user_back());
2975
2976 // HasShareds is true if any variables are captured in the outlined region,
2977 // false otherwise.
2978 bool HasShareds = StaleCI->arg_size() > 1;
2979 Builder.SetInsertPoint(StaleCI);
2980
2981 // Gather the arguments for emitting the runtime call for
2982 // @__kmpc_omp_task_alloc
2983 Function *TaskAllocFn =
2984 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_alloc);
2985
2986 // Arguments - `loc_ref` (Ident) and `gtid` (ThreadID)
2987 // call.
2988 Value *ThreadID = getOrCreateThreadID(Ident);
2989
2990 // Argument - `flags`
2991 // Task is tied iff (Flags & 1) == 1.
2992 // Task is untied iff (Flags & 1) == 0.
2993 // Task is final iff (Flags & 2) == 2.
2994 // Task is not final iff (Flags & 2) == 0.
2995 // Task is mergeable or merged-if0 iff (Flags & 4) == 4.
2996 // Task is neither mergeable nor merged-if0 iff (Flags & 4) == 0.
2997 // Task is detachable iff (Flags & 64) == 64.
2998 // Task is not detachable iff (Flags & 64) == 0.
2999 // Task is priority iff (Flags & 32) == 32.
3000 // Task is not priority iff (Flags & 32) == 0.
3001 // Task is free-agent eligible iff (Flags & 128) == 128.
3002 // Task is not free-agent eligible iff (Flags & 128) == 0.
3003 // TODO: Handle the other flags.
3004 Value *Flags = Builder.getInt32(C: Tied);
3005 auto *ConstIfCondition = dyn_cast_or_null<ConstantInt>(Val: IfCondition);
3006 bool UseMergedIf0Path = ConstIfCondition && ConstIfCondition->isZero();
3007 if (Final) {
3008 Value *FinalFlag =
3009 Builder.CreateSelect(C: Final, True: Builder.getInt32(C: 2), False: Builder.getInt32(C: 0));
3010 Flags = Builder.CreateOr(LHS: FinalFlag, RHS: Flags);
3011 }
3012
3013 if (Mergeable || UseMergedIf0Path)
3014 Flags = Builder.CreateOr(LHS: Builder.getInt32(C: 4), RHS: Flags);
3015 if (EventHandle)
3016 Flags = Builder.CreateOr(LHS: Builder.getInt32(C: 64), RHS: Flags);
3017 if (Priority)
3018 Flags = Builder.CreateOr(LHS: Builder.getInt32(C: 32), RHS: Flags);
3019 if (FreeAgent)
3020 Flags = Builder.CreateOr(LHS: Builder.getInt32(C: 128), RHS: Flags);
3021
3022 // Argument - `sizeof_kmp_task_t` (TaskSize)
3023 // Tasksize refers to the size in bytes of kmp_task_t data structure
3024 // including private vars accessed in task.
3025 // TODO: add kmp_task_t_with_privates (privates)
3026 Value *TaskSize = Builder.getInt64(
3027 C: divideCeil(Numerator: M.getDataLayout().getTypeSizeInBits(Ty: Task), Denominator: 8));
3028
3029 // Argument - `sizeof_shareds` (SharedsSize)
3030 // SharedsSize refers to the shareds array size in the kmp_task_t data
3031 // structure.
3032 Value *SharedsSize = Builder.getInt64(C: 0);
3033 if (HasShareds) {
3034 AllocaInst *ArgStructAlloca =
3035 dyn_cast<AllocaInst>(Val: StaleCI->getArgOperand(i: 1));
3036 assert(ArgStructAlloca &&
3037 "Unable to find the alloca instruction corresponding to arguments "
3038 "for extracted function");
3039 std::optional<TypeSize> ArgAllocSize =
3040 ArgStructAlloca->getAllocationSize(DL: M.getDataLayout());
3041 assert(ArgAllocSize &&
3042 "Unable to determine size of arguments for extracted function");
3043 SharedsSize = Builder.getInt64(C: ArgAllocSize->getFixedValue());
3044 }
3045 // Emit the @__kmpc_omp_task_alloc runtime call
3046 // The runtime call returns a pointer to an area where the task captured
3047 // variables must be copied before the task is run (TaskData)
3048 CallInst *TaskData = createRuntimeFunctionCall(
3049 Callee: TaskAllocFn, Args: {/*loc_ref=*/Ident, /*gtid=*/ThreadID, /*flags=*/Flags,
3050 /*sizeof_task=*/TaskSize, /*sizeof_shared=*/SharedsSize,
3051 /*task_func=*/&OutlinedFn});
3052
3053 if (Affinities.Count && Affinities.Info) {
3054 Function *RegAffFn = getOrCreateRuntimeFunctionPtr(
3055 FnID: OMPRTL___kmpc_omp_reg_task_with_affinity);
3056
3057 createRuntimeFunctionCall(Callee: RegAffFn, Args: {Ident, ThreadID, TaskData,
3058 Affinities.Count, Affinities.Info});
3059 }
3060
3061 // Emit detach clause initialization.
3062 // evt = (typeof(evt))__kmpc_task_allow_completion_event(loc, tid,
3063 // task_descriptor);
3064 if (EventHandle) {
3065 Function *TaskDetachFn = getOrCreateRuntimeFunctionPtr(
3066 FnID: OMPRTL___kmpc_task_allow_completion_event);
3067 llvm::Value *EventVal =
3068 createRuntimeFunctionCall(Callee: TaskDetachFn, Args: {Ident, ThreadID, TaskData});
3069 llvm::Value *EventHandleAddr =
3070 Builder.CreatePointerBitCastOrAddrSpaceCast(V: EventHandle,
3071 DestTy: Builder.getPtrTy(AddrSpace: 0));
3072 EventVal = Builder.CreatePtrToInt(V: EventVal, DestTy: Builder.getInt64Ty());
3073 Builder.CreateStore(Val: EventVal, Ptr: EventHandleAddr);
3074 }
3075 // Copy the arguments for outlined function
3076 if (HasShareds) {
3077 Value *Shareds = StaleCI->getArgOperand(i: 1);
3078 Align Alignment = TaskData->getPointerAlignment(DL: M.getDataLayout());
3079 Value *TaskShareds = Builder.CreateLoad(Ty: VoidPtr, Ptr: TaskData);
3080 Builder.CreateMemCpy(Dst: TaskShareds, DstAlign: Alignment, Src: Shareds, SrcAlign: Alignment,
3081 Size: SharedsSize);
3082 }
3083
3084 if (Priority) {
3085 //
3086 // The return type of "__kmpc_omp_task_alloc" is "kmp_task_t *",
3087 // we populate the priority information into the "kmp_task_t" here
3088 //
3089 // The struct "kmp_task_t" definition is available in kmp.h
3090 // kmp_task_t = { shareds, routine, part_id, data1, data2 }
3091 // data2 is used for priority
3092 //
3093 Type *Int32Ty = Builder.getInt32Ty();
3094 Constant *Zero = ConstantInt::get(Ty: Int32Ty, V: 0);
3095 // kmp_task_t* => { ptr }
3096 Type *TaskPtr = StructType::get(elt1: VoidPtr);
3097 Value *TaskGEP =
3098 Builder.CreateInBoundsGEP(Ty: TaskPtr, Ptr: TaskData, IdxList: {Zero, Zero});
3099 // kmp_task_t => { ptr, ptr, i32, ptr, ptr }
3100 Type *TaskStructType = StructType::get(
3101 elt1: VoidPtr, elts: VoidPtr, elts: Builder.getInt32Ty(), elts: VoidPtr, elts: VoidPtr);
3102 Value *PriorityData = Builder.CreateInBoundsGEP(
3103 Ty: TaskStructType, Ptr: TaskGEP, IdxList: {Zero, ConstantInt::get(Ty: Int32Ty, V: 4)});
3104 // kmp_cmplrdata_t => { ptr, ptr }
3105 Type *CmplrStructType = StructType::get(elt1: VoidPtr, elts: VoidPtr);
3106 Value *CmplrData = Builder.CreateInBoundsGEP(Ty: CmplrStructType,
3107 Ptr: PriorityData, IdxList: {Zero, Zero});
3108 Builder.CreateStore(Val: Priority, Ptr: CmplrData);
3109 }
3110
3111 Value *DepArray = nullptr;
3112 Value *NumDeps = nullptr;
3113 if (Dependencies.DepArray) {
3114 DepArray = Dependencies.DepArray;
3115 NumDeps = Dependencies.NumDeps;
3116 } else if (!Dependencies.Deps.empty()) {
3117 DepArray = emitTaskDependencies(OMPBuilder&: *this, Dependencies: Dependencies.Deps);
3118 NumDeps = Builder.getInt32(C: Dependencies.Deps.size());
3119 }
3120
3121 // In the presence of the `if` clause, the following IR is generated:
3122 // ...
3123 // %data = call @__kmpc_omp_task_alloc(...)
3124 // br i1 %if_condition, label %then, label %else
3125 // then:
3126 // call @__kmpc_omp_task(...)
3127 // br label %exit
3128 // else:
3129 // ;; Wait for resolution of dependencies, if any, before
3130 // ;; beginning the task
3131 // call @__kmpc_omp_wait_deps(...)
3132 // call @__kmpc_omp_task_begin_if0(...)
3133 // call @outlined_fn(...)
3134 // call @__kmpc_omp_task_complete_if0(...)
3135 // br label %exit
3136 // exit:
3137 // ...
3138 if (IfCondition && !UseMergedIf0Path) {
3139 // `SplitBlockAndInsertIfThenElse` requires the block to have a
3140 // terminator.
3141 splitBB(Builder, /*CreateBranch=*/true, Name: "if.end");
3142 Instruction *IfTerminator =
3143 Builder.GetInsertPoint()->getParent()->getTerminator();
3144 Instruction *ThenTI = IfTerminator, *ElseTI = nullptr;
3145 Builder.SetInsertPoint(IfTerminator);
3146 SplitBlockAndInsertIfThenElse(Cond: IfCondition, SplitBefore: IfTerminator, ThenTerm: &ThenTI,
3147 ElseTerm: &ElseTI);
3148 Builder.SetInsertPoint(ElseTI);
3149
3150 if (DepArray) {
3151 Function *TaskWaitFn =
3152 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_wait_deps);
3153 createRuntimeFunctionCall(
3154 Callee: TaskWaitFn,
3155 Args: {Ident, ThreadID, NumDeps, DepArray,
3156 ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0),
3157 ConstantPointerNull::get(T: PointerType::getUnqual(C&: M.getContext()))});
3158 }
3159 Function *TaskBeginFn =
3160 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_begin_if0);
3161 Function *TaskCompleteFn =
3162 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_complete_if0);
3163 createRuntimeFunctionCall(Callee: TaskBeginFn, Args: {Ident, ThreadID, TaskData});
3164 CallInst *CI = nullptr;
3165 if (HasShareds)
3166 CI = createRuntimeFunctionCall(Callee: &OutlinedFn, Args: {ThreadID, TaskData});
3167 else
3168 CI = createRuntimeFunctionCall(Callee: &OutlinedFn, Args: {ThreadID});
3169 CI->setDebugLoc(StaleCI->getDebugLoc());
3170 createRuntimeFunctionCall(Callee: TaskCompleteFn, Args: {Ident, ThreadID, TaskData});
3171 Builder.SetInsertPoint(ThenTI);
3172 }
3173
3174 if (DepArray) {
3175 Function *TaskFn =
3176 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_with_deps);
3177 createRuntimeFunctionCall(
3178 Callee: TaskFn,
3179 Args: {Ident, ThreadID, TaskData, NumDeps, DepArray,
3180 ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0),
3181 ConstantPointerNull::get(T: PointerType::getUnqual(C&: M.getContext()))});
3182
3183 } else {
3184 // Emit the @__kmpc_omp_task runtime call to spawn the task
3185 Function *TaskFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task);
3186 createRuntimeFunctionCall(Callee: TaskFn, Args: {Ident, ThreadID, TaskData});
3187 }
3188
3189 StaleCI->eraseFromParent();
3190
3191 Builder.SetInsertPoint(TheBB: TaskAllocaBB, IP: TaskAllocaBB->begin());
3192 if (HasShareds) {
3193 LoadInst *Shareds = Builder.CreateLoad(Ty: VoidPtr, Ptr: OutlinedFn.getArg(i: 1));
3194 OutlinedFn.getArg(i: 1)->replaceUsesWithIf(
3195 New: Shareds, ShouldReplace: [Shareds](Use &U) { return U.getUser() != Shareds; });
3196 }
3197
3198 // The insert point may refer to one of the instructions about to be
3199 // deleted. It is not needed anymore so clear it instead of leaving it
3200 // dangling.
3201 Builder.ClearInsertionPoint();
3202 for (Instruction *I : llvm::reverse(C&: ToBeDeleted))
3203 I->eraseFromParent();
3204 };
3205
3206 addOutlineInfo(OI: std::move(OI));
3207 Builder.SetInsertPoint(TheBB: TaskExitBB, IP: TaskExitBB->begin());
3208
3209 return Builder.saveIP();
3210}
3211
3212OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTaskgroup(
3213 const LocationDescription &Loc, InsertPointTy AllocaIP,
3214 ArrayRef<BasicBlock *> DeallocBlocks, BodyGenCallbackTy BodyGenCB) {
3215 if (!updateToLocation(Loc))
3216 return InsertPointTy();
3217
3218 uint32_t SrcLocStrSize;
3219 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
3220 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
3221 Value *ThreadID = getOrCreateThreadID(Ident);
3222
3223 // Emit the @__kmpc_taskgroup runtime call to start the taskgroup
3224 Function *TaskgroupFn =
3225 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_taskgroup);
3226 createRuntimeFunctionCall(Callee: TaskgroupFn, Args: {Ident, ThreadID});
3227
3228 BasicBlock *TaskgroupExitBB = splitBB(Builder, CreateBranch: true, Name: "taskgroup.exit");
3229 if (Error Err = BodyGenCB(AllocaIP, Builder.saveIP(), DeallocBlocks))
3230 return Err;
3231
3232 Builder.SetInsertPoint(TaskgroupExitBB);
3233 // Emit the @__kmpc_end_taskgroup runtime call to end the taskgroup
3234 Function *EndTaskgroupFn =
3235 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_end_taskgroup);
3236 createRuntimeFunctionCall(Callee: EndTaskgroupFn, Args: {Ident, ThreadID});
3237
3238 return Builder.saveIP();
3239}
3240
3241OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createSections(
3242 const LocationDescription &Loc, InsertPointTy AllocaIP,
3243 ArrayRef<StorableBodyGenCallbackTy> SectionCBs, PrivatizeCallbackTy PrivCB,
3244 FinalizeCallbackTy FiniCB, bool IsCancellable, bool IsNowait) {
3245 assert(!isConflictIP(AllocaIP, Loc.IP) && "Dedicated IP allocas required");
3246
3247 if (!updateToLocation(Loc))
3248 return Loc.IP;
3249
3250 FinalizationStack.push_back(Elt: {FiniCB, OMPD_sections, IsCancellable});
3251
3252 // Each section is emitted as a switch case
3253 // Each finalization callback is handled from clang.EmitOMPSectionDirective()
3254 // -> OMP.createSection() which generates the IR for each section
3255 // Iterate through all sections and emit a switch construct:
3256 // switch (IV) {
3257 // case 0:
3258 // <SectionStmt[0]>;
3259 // break;
3260 // ...
3261 // case <NumSection> - 1:
3262 // <SectionStmt[<NumSection> - 1]>;
3263 // break;
3264 // }
3265 // ...
3266 // section_loop.after:
3267 // <FiniCB>;
3268 auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, Value *IndVar) -> Error {
3269 Builder.restoreIP(IP: CodeGenIP);
3270 BasicBlock *Continue =
3271 splitBBWithSuffix(Builder, /*CreateBranch=*/false, Suffix: ".sections.after");
3272 Function *CurFn = Continue->getParent();
3273 SwitchInst *SwitchStmt = Builder.CreateSwitch(V: IndVar, Dest: Continue);
3274
3275 unsigned CaseNumber = 0;
3276 for (auto SectionCB : SectionCBs) {
3277 BasicBlock *CaseBB = BasicBlock::Create(
3278 Context&: M.getContext(), Name: "omp_section_loop.body.case", Parent: CurFn, InsertBefore: Continue);
3279 SwitchStmt->addCase(OnVal: Builder.getInt32(C: CaseNumber), Dest: CaseBB);
3280 Builder.SetInsertPoint(CaseBB);
3281 UncondBrInst *CaseEndBr = Builder.CreateBr(Dest: Continue);
3282 if (Error Err =
3283 SectionCB(InsertPointTy(),
3284 {CaseEndBr->getParent(), CaseEndBr->getIterator()}, {}))
3285 return Err;
3286 CaseNumber++;
3287 }
3288 // remove the existing terminator from body BB since there can be no
3289 // terminators after switch/case
3290 return Error::success();
3291 };
3292 // Loop body ends here
3293 // LowerBound, UpperBound, and STride for createCanonicalLoop
3294 Type *I32Ty = Type::getInt32Ty(C&: M.getContext());
3295 Value *LB = ConstantInt::get(Ty: I32Ty, V: 0);
3296 Value *UB = ConstantInt::get(Ty: I32Ty, V: SectionCBs.size());
3297 Value *ST = ConstantInt::get(Ty: I32Ty, V: 1);
3298 Expected<CanonicalLoopInfo *> LoopInfo = createCanonicalLoop(
3299 Loc, BodyGenCB: LoopBodyGenCB, Start: LB, Stop: UB, Step: ST, IsSigned: true, InclusiveStop: false, ComputeIP: AllocaIP, Name: "section_loop");
3300 if (!LoopInfo)
3301 return LoopInfo.takeError();
3302
3303 InsertPointOrErrorTy WsloopIP =
3304 applyStaticWorkshareLoop(DL: Loc.DL, CLI: *LoopInfo, AllocaIP,
3305 LoopType: WorksharingLoopType::ForStaticLoop, NeedsBarrier: !IsNowait);
3306 if (!WsloopIP)
3307 return WsloopIP.takeError();
3308 InsertPointTy AfterIP = *WsloopIP;
3309
3310 BasicBlock *LoopFini = AfterIP.getBlock()->getSinglePredecessor();
3311 assert(LoopFini && "Bad structure of static workshare loop finalization");
3312
3313 // Apply the finalization callback in LoopAfterBB
3314 auto FiniInfo = FinalizationStack.pop_back_val();
3315 assert(FiniInfo.DK == OMPD_sections &&
3316 "Unexpected finalization stack state!");
3317 if (Error Err = FiniInfo.mergeFiniBB(Builder, OtherFiniBB: LoopFini))
3318 return Err;
3319
3320 return AfterIP;
3321}
3322
3323OpenMPIRBuilder::InsertPointOrErrorTy
3324OpenMPIRBuilder::createSection(const LocationDescription &Loc,
3325 BodyGenCallbackTy BodyGenCB,
3326 FinalizeCallbackTy FiniCB) {
3327 if (!updateToLocation(Loc))
3328 return Loc.IP;
3329
3330 auto FiniCBWrapper = [&](InsertPointTy IP) {
3331 if (IP.getBlock()->end() != IP.getPoint())
3332 return FiniCB(IP);
3333 // This must be done otherwise any nested constructs using FinalizeOMPRegion
3334 // will fail because that function requires the Finalization Basic Block to
3335 // have a terminator, which is already removed by EmitOMPRegionBody.
3336 // IP is currently at cancelation block.
3337 // We need to backtrack to the condition block to fetch
3338 // the exit block and create a branch from cancelation
3339 // to exit block.
3340 IRBuilder<>::InsertPointGuard IPG(Builder);
3341 Builder.restoreIP(IP);
3342 auto *CaseBB = Loc.IP.getBlock();
3343 auto *CondBB = CaseBB->getSinglePredecessor()->getSinglePredecessor();
3344 auto *ExitBB = CondBB->getTerminator()->getSuccessor(Idx: 1);
3345 Instruction *I = Builder.CreateBr(Dest: ExitBB);
3346 IP = InsertPointTy(I->getParent(), I->getIterator());
3347 return FiniCB(IP);
3348 };
3349
3350 Directive OMPD = Directive::OMPD_sections;
3351 // Since we are using Finalization Callback here, HasFinalize
3352 // and IsCancellable have to be true
3353 return EmitOMPInlinedRegion(OMPD, EntryCall: nullptr, ExitCall: nullptr, BodyGenCB, FiniCB: FiniCBWrapper,
3354 /*Conditional*/ false, /*hasFinalize*/ HasFinalize: true,
3355 /*IsCancellable*/ true);
3356}
3357
3358static OpenMPIRBuilder::InsertPointTy getInsertPointAfterInstr(Instruction *I) {
3359 BasicBlock::iterator IT(I);
3360 IT++;
3361 return OpenMPIRBuilder::InsertPointTy(I->getParent(), IT);
3362}
3363
3364Value *OpenMPIRBuilder::getGPUThreadID() {
3365 return createRuntimeFunctionCall(
3366 Callee: getOrCreateRuntimeFunction(M,
3367 FnID: OMPRTL___kmpc_get_hardware_thread_id_in_block),
3368 Args: {});
3369}
3370
3371Value *OpenMPIRBuilder::getGPUWarpSize() {
3372 return createRuntimeFunctionCall(
3373 Callee: getOrCreateRuntimeFunction(M, FnID: OMPRTL___kmpc_get_warp_size), Args: {});
3374}
3375
3376Value *OpenMPIRBuilder::getNVPTXWarpID() {
3377 unsigned LaneIDBits = Log2_32(Value: Config.getGridValue().GV_Warp_Size);
3378 return Builder.CreateAShr(LHS: getGPUThreadID(), RHS: LaneIDBits, Name: "nvptx_warp_id");
3379}
3380
3381Value *OpenMPIRBuilder::getNVPTXLaneID() {
3382 unsigned LaneIDBits = Log2_32(Value: Config.getGridValue().GV_Warp_Size);
3383 assert(LaneIDBits < 32 && "Invalid LaneIDBits size in NVPTX device.");
3384 unsigned LaneIDMask = ~0u >> (32u - LaneIDBits);
3385 return Builder.CreateAnd(LHS: getGPUThreadID(), RHS: Builder.getInt32(C: LaneIDMask),
3386 Name: "nvptx_lane_id");
3387}
3388
3389Value *OpenMPIRBuilder::castValueToType(InsertPointTy AllocaIP, Value *From,
3390 Type *ToType) {
3391 Type *FromType = From->getType();
3392 uint64_t FromSize = M.getDataLayout().getTypeStoreSize(Ty: FromType);
3393 uint64_t ToSize = M.getDataLayout().getTypeStoreSize(Ty: ToType);
3394 assert(FromSize > 0 && "From size must be greater than zero");
3395 assert(ToSize > 0 && "To size must be greater than zero");
3396 if (FromType == ToType)
3397 return From;
3398 if (FromSize == ToSize)
3399 return Builder.CreateBitCast(V: From, DestTy: ToType);
3400 if (ToType->isIntegerTy() && FromType->isIntegerTy())
3401 return Builder.CreateIntCast(V: From, DestTy: ToType, /*isSigned*/ true);
3402 InsertPointTy SaveIP = Builder.saveIP();
3403 Builder.restoreIP(IP: AllocaIP);
3404 Value *CastItem = Builder.CreateAlloca(Ty: ToType);
3405 Builder.restoreIP(IP: SaveIP);
3406
3407 Value *ValCastItem = Builder.CreatePointerBitCastOrAddrSpaceCast(
3408 V: CastItem, DestTy: Builder.getPtrTy(AddrSpace: 0));
3409 Builder.CreateStore(Val: From, Ptr: ValCastItem);
3410 return Builder.CreateLoad(Ty: ToType, Ptr: CastItem);
3411}
3412
3413Value *OpenMPIRBuilder::createRuntimeShuffleFunction(InsertPointTy AllocaIP,
3414 Value *Element,
3415 Type *ElementType,
3416 Value *Offset) {
3417 uint64_t Size = M.getDataLayout().getTypeStoreSize(Ty: ElementType);
3418 assert(Size <= 8 && "Unsupported bitwidth in shuffle instruction");
3419
3420 // Cast all types to 32- or 64-bit values before calling shuffle routines.
3421 Type *CastTy = Builder.getIntNTy(N: Size <= 4 ? 32 : 64);
3422 Value *ElemCast = castValueToType(AllocaIP, From: Element, ToType: CastTy);
3423 Value *WarpSize =
3424 Builder.CreateIntCast(V: getGPUWarpSize(), DestTy: Builder.getInt16Ty(), isSigned: true);
3425 Function *ShuffleFunc = getOrCreateRuntimeFunctionPtr(
3426 FnID: Size <= 4 ? RuntimeFunction::OMPRTL___kmpc_shuffle_int32
3427 : RuntimeFunction::OMPRTL___kmpc_shuffle_int64);
3428 Value *WarpSizeCast =
3429 Builder.CreateIntCast(V: WarpSize, DestTy: Builder.getInt16Ty(), /*isSigned=*/true);
3430 Value *ShuffleCall =
3431 createRuntimeFunctionCall(Callee: ShuffleFunc, Args: {ElemCast, Offset, WarpSizeCast});
3432 // The shuffle runtime functions return a 32- or 64-bit value. Cast it back
3433 // down to the requested element type, otherwise storing the result would
3434 // write past the end of an element narrower than the shuffle width.
3435 return castValueToType(AllocaIP, From: ShuffleCall, ToType: ElementType);
3436}
3437
3438void OpenMPIRBuilder::shuffleAndStore(InsertPointTy AllocaIP, Value *SrcAddr,
3439 Value *DstAddr, Type *ElemType,
3440 Value *Offset, Type *ReductionArrayTy,
3441 bool IsByRefElem) {
3442 uint64_t Size = M.getDataLayout().getTypeStoreSize(Ty: ElemType);
3443 // Create the loop over the big sized data.
3444 // ptr = (void*)Elem;
3445 // ptrEnd = (void*) Elem + 1;
3446 // Step = 8;
3447 // while (ptr + Step < ptrEnd)
3448 // shuffle((int64_t)*ptr);
3449 // Step = 4;
3450 // while (ptr + Step < ptrEnd)
3451 // shuffle((int32_t)*ptr);
3452 // ...
3453 Type *IndexTy = Builder.getIndexTy(
3454 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
3455 Value *ElemPtr = DstAddr;
3456 Value *Ptr = SrcAddr;
3457 for (unsigned IntSize = 8; IntSize >= 1; IntSize /= 2) {
3458 if (Size < IntSize)
3459 continue;
3460 Type *IntType = Builder.getIntNTy(N: IntSize * 8);
3461 Ptr = Builder.CreatePointerBitCastOrAddrSpaceCast(
3462 V: Ptr, DestTy: Builder.getPtrTy(AddrSpace: 0), Name: Ptr->getName() + ".ascast");
3463 Value *SrcAddrGEP =
3464 Builder.CreateGEP(Ty: ElemType, Ptr: SrcAddr, IdxList: {ConstantInt::get(Ty: IndexTy, V: 1)});
3465 ElemPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
3466 V: ElemPtr, DestTy: Builder.getPtrTy(AddrSpace: 0), Name: ElemPtr->getName() + ".ascast");
3467
3468 Function *CurFunc = Builder.GetInsertBlock()->getParent();
3469 if ((Size / IntSize) > 1) {
3470 Value *PtrEnd = Builder.CreatePointerBitCastOrAddrSpaceCast(
3471 V: SrcAddrGEP, DestTy: Builder.getPtrTy());
3472 BasicBlock *PreCondBB =
3473 BasicBlock::Create(Context&: M.getContext(), Name: ".shuffle.pre_cond");
3474 BasicBlock *ThenBB = BasicBlock::Create(Context&: M.getContext(), Name: ".shuffle.then");
3475 BasicBlock *ExitBB = BasicBlock::Create(Context&: M.getContext(), Name: ".shuffle.exit");
3476 BasicBlock *CurrentBB = Builder.GetInsertBlock();
3477 emitBlock(BB: PreCondBB, CurFn: CurFunc);
3478 PHINode *PhiSrc =
3479 Builder.CreatePHI(Ty: Ptr->getType(), /*NumReservedValues=*/2);
3480 PhiSrc->addIncoming(V: Ptr, BB: CurrentBB);
3481 PHINode *PhiDest =
3482 Builder.CreatePHI(Ty: ElemPtr->getType(), /*NumReservedValues=*/2);
3483 PhiDest->addIncoming(V: ElemPtr, BB: CurrentBB);
3484 Ptr = PhiSrc;
3485 ElemPtr = PhiDest;
3486 Value *PtrDiff = Builder.CreatePtrDiff(
3487 ElemTy: Builder.getInt8Ty(), LHS: PtrEnd,
3488 RHS: Builder.CreatePointerBitCastOrAddrSpaceCast(V: Ptr, DestTy: Builder.getPtrTy()));
3489 Builder.CreateCondBr(
3490 Cond: Builder.CreateICmpSGT(LHS: PtrDiff, RHS: Builder.getInt64(C: IntSize - 1)), True: ThenBB,
3491 False: ExitBB);
3492 emitBlock(BB: ThenBB, CurFn: CurFunc);
3493 Value *Res = createRuntimeShuffleFunction(
3494 AllocaIP,
3495 Element: Builder.CreateAlignedLoad(
3496 Ty: IntType, Ptr, Align: M.getDataLayout().getPrefTypeAlign(Ty: ElemType)),
3497 ElementType: IntType, Offset);
3498 Builder.CreateAlignedStore(Val: Res, Ptr: ElemPtr,
3499 Align: M.getDataLayout().getPrefTypeAlign(Ty: ElemType));
3500 Value *LocalPtr =
3501 Builder.CreateGEP(Ty: IntType, Ptr, IdxList: {ConstantInt::get(Ty: IndexTy, V: 1)});
3502 Value *LocalElemPtr =
3503 Builder.CreateGEP(Ty: IntType, Ptr: ElemPtr, IdxList: {ConstantInt::get(Ty: IndexTy, V: 1)});
3504 PhiSrc->addIncoming(V: LocalPtr, BB: ThenBB);
3505 PhiDest->addIncoming(V: LocalElemPtr, BB: ThenBB);
3506 emitBranch(Target: PreCondBB);
3507 emitBlock(BB: ExitBB, CurFn: CurFunc);
3508 } else {
3509 // The shuffled value comes back as the chunk's integer type, so the
3510 // store covers exactly this chunk regardless of what ElemType is.
3511 Value *Res = createRuntimeShuffleFunction(
3512 AllocaIP, Element: Builder.CreateLoad(Ty: IntType, Ptr), ElementType: IntType, Offset);
3513 Builder.CreateStore(Val: Res, Ptr: ElemPtr);
3514 Ptr = Builder.CreateGEP(Ty: IntType, Ptr, IdxList: {ConstantInt::get(Ty: IndexTy, V: 1)});
3515 ElemPtr =
3516 Builder.CreateGEP(Ty: IntType, Ptr: ElemPtr, IdxList: {ConstantInt::get(Ty: IndexTy, V: 1)});
3517 }
3518 Size = Size % IntSize;
3519 }
3520}
3521
3522Error OpenMPIRBuilder::emitReductionListCopy(
3523 InsertPointTy AllocaIP, CopyAction Action, Type *ReductionArrayTy,
3524 ArrayRef<ReductionInfo> ReductionInfos, Value *SrcBase, Value *DestBase,
3525 ArrayRef<bool> IsByRef, CopyOptionsTy CopyOptions) {
3526 Type *IndexTy = Builder.getIndexTy(
3527 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
3528 Value *RemoteLaneOffset = CopyOptions.RemoteLaneOffset;
3529
3530 // Iterates, element-by-element, through the source Reduce list and
3531 // make a copy.
3532 for (auto En : enumerate(First&: ReductionInfos)) {
3533 const ReductionInfo &RI = En.value();
3534 Value *SrcElementAddr = nullptr;
3535 AllocaInst *DestAlloca = nullptr;
3536 Value *DestElementAddr = nullptr;
3537 Value *DestElementPtrAddr = nullptr;
3538 // Should we shuffle in an element from a remote lane?
3539 bool ShuffleInElement = false;
3540 // Set to true to update the pointer in the dest Reduce list to a
3541 // newly created element.
3542 bool UpdateDestListPtr = false;
3543
3544 // Step 1.1: Get the address for the src element in the Reduce list.
3545 Value *SrcElementPtrAddr = Builder.CreateInBoundsGEP(
3546 Ty: ReductionArrayTy, Ptr: SrcBase,
3547 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
3548 SrcElementAddr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: SrcElementPtrAddr);
3549
3550 // Step 1.2: Create a temporary to store the element in the destination
3551 // Reduce list.
3552 DestElementPtrAddr = Builder.CreateInBoundsGEP(
3553 Ty: ReductionArrayTy, Ptr: DestBase,
3554 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
3555 bool IsByRefElem = (!IsByRef.empty() && IsByRef[En.index()]);
3556 switch (Action) {
3557 case CopyAction::RemoteLaneToThread: {
3558 InsertPointTy CurIP = Builder.saveIP();
3559 Builder.restoreIP(IP: AllocaIP);
3560
3561 Type *DestAllocaType =
3562 IsByRefElem ? RI.ByRefAllocatedType : RI.ElementType;
3563 DestAlloca = Builder.CreateAlloca(Ty: DestAllocaType, ArraySize: nullptr,
3564 Name: ".omp.reduction.element");
3565 DestAlloca->setAlignment(
3566 M.getDataLayout().getPrefTypeAlign(Ty: DestAllocaType));
3567 DestElementAddr = DestAlloca;
3568 DestElementAddr =
3569 Builder.CreateAddrSpaceCast(V: DestElementAddr, DestTy: Builder.getPtrTy(),
3570 Name: DestElementAddr->getName() + ".ascast");
3571 Builder.restoreIP(IP: CurIP);
3572 ShuffleInElement = true;
3573 UpdateDestListPtr = true;
3574 break;
3575 }
3576 case CopyAction::ThreadCopy: {
3577 DestElementAddr =
3578 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: DestElementPtrAddr);
3579 break;
3580 }
3581 }
3582
3583 // Now that all active lanes have read the element in the
3584 // Reduce list, shuffle over the value from the remote lane.
3585 if (ShuffleInElement) {
3586 Type *ShuffleType = RI.ElementType;
3587 Value *ShuffleSrcAddr = SrcElementAddr;
3588 Value *ShuffleDestAddr = DestElementAddr;
3589 AllocaInst *LocalStorage = nullptr;
3590
3591 if (IsByRefElem) {
3592 assert(RI.ByRefElementType && "Expected by-ref element type to be set");
3593 assert(RI.ByRefAllocatedType &&
3594 "Expected by-ref allocated type to be set");
3595 // For by-ref reductions, we need to copy from the remote lane the
3596 // actual value of the partial reduction computed by that remote lane;
3597 // rather than, for example, a pointer to that data or, even worse, a
3598 // pointer to the descriptor of the by-ref reduction element.
3599 ShuffleType = RI.ByRefElementType;
3600
3601 if (RI.DataPtrPtrGen) {
3602 // Descriptor-based by-ref: extract data pointer from descriptor.
3603 InsertPointOrErrorTy GenResult = RI.DataPtrPtrGen(
3604 Builder.saveIP(), ShuffleSrcAddr, ShuffleSrcAddr);
3605
3606 if (!GenResult)
3607 return GenResult.takeError();
3608
3609 ShuffleSrcAddr =
3610 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ShuffleSrcAddr);
3611
3612 {
3613 InsertPointTy OldIP = Builder.saveIP();
3614 Builder.restoreIP(IP: AllocaIP);
3615
3616 LocalStorage = Builder.CreateAlloca(Ty: ShuffleType);
3617 Builder.restoreIP(IP: OldIP);
3618 ShuffleDestAddr = LocalStorage;
3619 }
3620 } else {
3621 // Non-descriptor by-ref: the pointer already references data
3622 // directly. Shuffle into the destination alloca.
3623 ShuffleDestAddr = DestElementAddr;
3624 }
3625 }
3626
3627 shuffleAndStore(AllocaIP, SrcAddr: ShuffleSrcAddr, DstAddr: ShuffleDestAddr, ElemType: ShuffleType,
3628 Offset: RemoteLaneOffset, ReductionArrayTy, IsByRefElem);
3629
3630 if (IsByRefElem && RI.DataPtrPtrGen) {
3631 // Copy descriptor from source and update base_ptr to shuffled data
3632 Value *DestDescriptorAddr = Builder.CreatePointerBitCastOrAddrSpaceCast(
3633 V: DestAlloca, DestTy: Builder.getPtrTy(), Name: ".ascast");
3634
3635 InsertPointOrErrorTy GenResult = generateReductionDescriptor(
3636 DescriptorAddr: DestDescriptorAddr, DataPtr: LocalStorage, SrcDescriptorAddr: SrcElementAddr,
3637 DescriptorType: RI.ByRefAllocatedType, DataPtrPtrGen: RI.DataPtrPtrGen);
3638
3639 if (!GenResult)
3640 return GenResult.takeError();
3641 }
3642 } else {
3643 switch (RI.EvaluationKind) {
3644 case EvalKind::Scalar: {
3645 Value *Elem = Builder.CreateLoad(Ty: RI.ElementType, Ptr: SrcElementAddr);
3646 // Store the source element value to the dest element address.
3647 Builder.CreateStore(Val: Elem, Ptr: DestElementAddr);
3648 break;
3649 }
3650 case EvalKind::Complex: {
3651 Value *SrcRealPtr = Builder.CreateConstInBoundsGEP2_32(
3652 Ty: RI.ElementType, Ptr: SrcElementAddr, Idx0: 0, Idx1: 0, Name: ".realp");
3653 Value *SrcReal = Builder.CreateLoad(
3654 Ty: RI.ElementType->getStructElementType(N: 0), Ptr: SrcRealPtr, Name: ".real");
3655 Value *SrcImgPtr = Builder.CreateConstInBoundsGEP2_32(
3656 Ty: RI.ElementType, Ptr: SrcElementAddr, Idx0: 0, Idx1: 1, Name: ".imagp");
3657 Value *SrcImg = Builder.CreateLoad(
3658 Ty: RI.ElementType->getStructElementType(N: 1), Ptr: SrcImgPtr, Name: ".imag");
3659
3660 Value *DestRealPtr = Builder.CreateConstInBoundsGEP2_32(
3661 Ty: RI.ElementType, Ptr: DestElementAddr, Idx0: 0, Idx1: 0, Name: ".realp");
3662 Value *DestImgPtr = Builder.CreateConstInBoundsGEP2_32(
3663 Ty: RI.ElementType, Ptr: DestElementAddr, Idx0: 0, Idx1: 1, Name: ".imagp");
3664 Builder.CreateStore(Val: SrcReal, Ptr: DestRealPtr);
3665 Builder.CreateStore(Val: SrcImg, Ptr: DestImgPtr);
3666 break;
3667 }
3668 case EvalKind::Aggregate: {
3669 Value *SizeVal = Builder.getInt64(
3670 C: M.getDataLayout().getTypeStoreSize(Ty: RI.ElementType));
3671 Builder.CreateMemCpy(
3672 Dst: DestElementAddr, DstAlign: M.getDataLayout().getPrefTypeAlign(Ty: RI.ElementType),
3673 Src: SrcElementAddr, SrcAlign: M.getDataLayout().getPrefTypeAlign(Ty: RI.ElementType),
3674 Size: SizeVal, isVolatile: false);
3675 break;
3676 }
3677 };
3678 }
3679
3680 // Step 3.1: Modify reference in dest Reduce list as needed.
3681 // Modifying the reference in Reduce list to point to the newly
3682 // created element. The element is live in the current function
3683 // scope and that of functions it invokes (i.e., reduce_function).
3684 // RemoteReduceData[i] = (void*)&RemoteElem
3685 if (UpdateDestListPtr) {
3686 Value *CastDestAddr = Builder.CreatePointerBitCastOrAddrSpaceCast(
3687 V: DestElementAddr, DestTy: Builder.getPtrTy(),
3688 Name: DestElementAddr->getName() + ".ascast");
3689 Builder.CreateStore(Val: CastDestAddr, Ptr: DestElementPtrAddr);
3690 }
3691 }
3692
3693 return Error::success();
3694}
3695
3696Expected<Function *> OpenMPIRBuilder::emitInterWarpCopyFunction(
3697 const LocationDescription &Loc, ArrayRef<ReductionInfo> ReductionInfos,
3698 AttributeList FuncAttrs, ArrayRef<bool> IsByRef) {
3699 IRBuilder<>::InsertPointGuard IPG(Builder);
3700 LLVMContext &Ctx = M.getContext();
3701 FunctionType *FuncTy = FunctionType::get(
3702 Result: Builder.getVoidTy(), Params: {Builder.getPtrTy(), Builder.getInt32Ty()},
3703 /* IsVarArg */ isVarArg: false);
3704 Function *WcFunc =
3705 Function::Create(Ty: FuncTy, Linkage: GlobalVariable::InternalLinkage,
3706 N: "_omp_reduction_inter_warp_copy_func", M: &M);
3707 WcFunc->setCallingConv(Config.getRuntimeCC());
3708 WcFunc->setAttributes(FuncAttrs);
3709 WcFunc->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
3710 WcFunc->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
3711 BasicBlock *EntryBB = BasicBlock::Create(Context&: M.getContext(), Name: "entry", Parent: WcFunc);
3712 Builder.SetInsertPoint(EntryBB);
3713 Builder.SetCurrentDebugLocation(llvm::DebugLoc());
3714
3715 // ReduceList: thread local Reduce list.
3716 // At the stage of the computation when this function is called, partially
3717 // aggregated values reside in the first lane of every active warp.
3718 Argument *ReduceListArg = WcFunc->getArg(i: 0);
3719 // NumWarps: number of warps active in the parallel region. This could
3720 // be smaller than 32 (max warps in a CTA) for partial block reduction.
3721 Argument *NumWarpsArg = WcFunc->getArg(i: 1);
3722
3723 // This array is used as a medium to transfer, one reduce element at a time,
3724 // the data from the first lane of every warp to lanes in the first warp
3725 // in order to perform the final step of a reduction in a parallel region
3726 // (reduction across warps). The array is placed in NVPTX __shared__ memory
3727 // for reduced latency, as well as to have a distinct copy for concurrently
3728 // executing target regions. The array is declared with common linkage so
3729 // as to be shared across compilation units.
3730 StringRef TransferMediumName =
3731 "__openmp_nvptx_data_transfer_temporary_storage";
3732 GlobalVariable *TransferMedium = M.getGlobalVariable(Name: TransferMediumName);
3733 unsigned WarpSize = Config.getGridValue().GV_Warp_Size;
3734 ArrayType *ArrayTy = ArrayType::get(ElementType: Builder.getInt32Ty(), NumElements: WarpSize);
3735 if (!TransferMedium) {
3736 TransferMedium = new GlobalVariable(
3737 M, ArrayTy, /*isConstant=*/false, GlobalVariable::WeakAnyLinkage,
3738 UndefValue::get(T: ArrayTy), TransferMediumName,
3739 /*InsertBefore=*/nullptr, GlobalVariable::NotThreadLocal,
3740 /*AddressSpace=*/3);
3741 }
3742
3743 // Get the CUDA thread id of the current OpenMP thread on the GPU.
3744 Value *GPUThreadID = getGPUThreadID();
3745 // nvptx_lane_id = nvptx_id % warpsize
3746 Value *LaneID = getNVPTXLaneID();
3747 // nvptx_warp_id = nvptx_id / warpsize
3748 Value *WarpID = getNVPTXWarpID();
3749
3750 InsertPointTy AllocaIP =
3751 InsertPointTy(Builder.GetInsertBlock(),
3752 Builder.GetInsertBlock()->getFirstInsertionPt());
3753 Type *Arg0Type = ReduceListArg->getType();
3754 Type *Arg1Type = NumWarpsArg->getType();
3755 Builder.restoreIP(IP: AllocaIP);
3756 AllocaInst *ReduceListAlloca = Builder.CreateAlloca(
3757 Ty: Arg0Type, ArraySize: nullptr, Name: ReduceListArg->getName() + ".addr");
3758 AllocaInst *NumWarpsAlloca =
3759 Builder.CreateAlloca(Ty: Arg1Type, ArraySize: nullptr, Name: NumWarpsArg->getName() + ".addr");
3760 Value *ReduceListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
3761 V: ReduceListAlloca, DestTy: Arg0Type, Name: ReduceListAlloca->getName() + ".ascast");
3762 Value *NumWarpsAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
3763 V: NumWarpsAlloca, DestTy: Builder.getPtrTy(AddrSpace: 0),
3764 Name: NumWarpsAlloca->getName() + ".ascast");
3765 Builder.CreateStore(Val: ReduceListArg, Ptr: ReduceListAddrCast);
3766 Builder.CreateStore(Val: NumWarpsArg, Ptr: NumWarpsAddrCast);
3767 AllocaIP = getInsertPointAfterInstr(I: NumWarpsAlloca);
3768 InsertPointTy CodeGenIP =
3769 getInsertPointAfterInstr(I: &Builder.GetInsertBlock()->back());
3770 Builder.restoreIP(IP: CodeGenIP);
3771
3772 Value *ReduceList =
3773 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ReduceListAddrCast);
3774
3775 for (auto En : enumerate(First&: ReductionInfos)) {
3776 //
3777 // Warp master copies reduce element to transfer medium in __shared__
3778 // memory.
3779 //
3780 const ReductionInfo &RI = En.value();
3781 bool IsByRefElem = !IsByRef.empty() && IsByRef[En.index()];
3782 unsigned RealTySize = M.getDataLayout().getTypeAllocSize(
3783 Ty: IsByRefElem ? RI.ByRefElementType : RI.ElementType);
3784 for (unsigned TySize = 4; TySize > 0 && RealTySize > 0; TySize /= 2) {
3785 Type *CType = Builder.getIntNTy(N: TySize * 8);
3786
3787 unsigned NumIters = RealTySize / TySize;
3788 if (NumIters == 0)
3789 continue;
3790 Value *Cnt = nullptr;
3791 Value *CntAddr = nullptr;
3792 BasicBlock *PrecondBB = nullptr;
3793 BasicBlock *ExitBB = nullptr;
3794 if (NumIters > 1) {
3795 CodeGenIP = Builder.saveIP();
3796 Builder.restoreIP(IP: AllocaIP);
3797 CntAddr =
3798 Builder.CreateAlloca(Ty: Builder.getInt32Ty(), ArraySize: nullptr, Name: ".cnt.addr");
3799
3800 CntAddr = Builder.CreateAddrSpaceCast(V: CntAddr, DestTy: Builder.getPtrTy(),
3801 Name: CntAddr->getName() + ".ascast");
3802 Builder.restoreIP(IP: CodeGenIP);
3803 Builder.CreateStore(Val: Constant::getNullValue(Ty: Builder.getInt32Ty()),
3804 Ptr: CntAddr,
3805 /*Volatile=*/isVolatile: false);
3806 PrecondBB = BasicBlock::Create(Context&: Ctx, Name: "precond");
3807 ExitBB = BasicBlock::Create(Context&: Ctx, Name: "exit");
3808 BasicBlock *BodyBB = BasicBlock::Create(Context&: Ctx, Name: "body");
3809 emitBlock(BB: PrecondBB, CurFn: Builder.GetInsertBlock()->getParent());
3810 Cnt = Builder.CreateLoad(Ty: Builder.getInt32Ty(), Ptr: CntAddr,
3811 /*Volatile=*/isVolatile: false);
3812 Value *Cmp = Builder.CreateICmpULT(
3813 LHS: Cnt, RHS: ConstantInt::get(Ty: Builder.getInt32Ty(), V: NumIters));
3814 Builder.CreateCondBr(Cond: Cmp, True: BodyBB, False: ExitBB);
3815 emitBlock(BB: BodyBB, CurFn: Builder.GetInsertBlock()->getParent());
3816 }
3817
3818 // kmpc_barrier.
3819 InsertPointOrErrorTy BarrierIP1 =
3820 createBarrier(Loc: LocationDescription(Builder.saveIP(), DebugLoc()),
3821 Kind: omp::Directive::OMPD_unknown,
3822 /* ForceSimpleCall */ false,
3823 /* CheckCancelFlag */ true);
3824 if (!BarrierIP1)
3825 return BarrierIP1.takeError();
3826 BasicBlock *ThenBB = BasicBlock::Create(Context&: Ctx, Name: "then");
3827 BasicBlock *ElseBB = BasicBlock::Create(Context&: Ctx, Name: "else");
3828 BasicBlock *MergeBB = BasicBlock::Create(Context&: Ctx, Name: "ifcont");
3829
3830 // if (lane_id == 0)
3831 Value *IsWarpMaster = Builder.CreateIsNull(Arg: LaneID, Name: "warp_master");
3832 Builder.CreateCondBr(Cond: IsWarpMaster, True: ThenBB, False: ElseBB);
3833 emitBlock(BB: ThenBB, CurFn: Builder.GetInsertBlock()->getParent());
3834
3835 // Reduce element = LocalReduceList[i]
3836 auto *RedListArrayTy =
3837 ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: ReductionInfos.size());
3838 Type *IndexTy = Builder.getIndexTy(
3839 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
3840 Value *ElemPtrPtr =
3841 Builder.CreateInBoundsGEP(Ty: RedListArrayTy, Ptr: ReduceList,
3842 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0),
3843 ConstantInt::get(Ty: IndexTy, V: En.index())});
3844 // elemptr = ((CopyType*)(elemptrptr)) + I
3845 Value *ElemPtr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ElemPtrPtr);
3846
3847 if (IsByRefElem && RI.DataPtrPtrGen) {
3848 InsertPointOrErrorTy GenRes =
3849 RI.DataPtrPtrGen(Builder.saveIP(), ElemPtr, ElemPtr);
3850
3851 if (!GenRes)
3852 return GenRes.takeError();
3853
3854 ElemPtr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ElemPtr);
3855 }
3856
3857 if (NumIters > 1)
3858 ElemPtr = Builder.CreateGEP(Ty: Builder.getInt32Ty(), Ptr: ElemPtr, IdxList: Cnt);
3859
3860 // Get pointer to location in transfer medium.
3861 // MediumPtr = &medium[warp_id]
3862 Value *MediumPtr = Builder.CreateInBoundsGEP(
3863 Ty: ArrayTy, Ptr: TransferMedium, IdxList: {Builder.getInt64(C: 0), WarpID});
3864 // elem = *elemptr
3865 //*MediumPtr = elem
3866 Value *Elem = Builder.CreateLoad(Ty: CType, Ptr: ElemPtr);
3867 // Store the source element value to the dest element address.
3868 Builder.CreateStore(Val: Elem, Ptr: MediumPtr,
3869 /*IsVolatile*/ isVolatile: true);
3870 Builder.CreateBr(Dest: MergeBB);
3871
3872 // else
3873 emitBlock(BB: ElseBB, CurFn: Builder.GetInsertBlock()->getParent());
3874 Builder.CreateBr(Dest: MergeBB);
3875
3876 // endif
3877 emitBlock(BB: MergeBB, CurFn: Builder.GetInsertBlock()->getParent());
3878 InsertPointOrErrorTy BarrierIP2 =
3879 createBarrier(Loc: LocationDescription(Builder.saveIP(), DebugLoc()),
3880 Kind: omp::Directive::OMPD_unknown,
3881 /* ForceSimpleCall */ false,
3882 /* CheckCancelFlag */ true);
3883 if (!BarrierIP2)
3884 return BarrierIP2.takeError();
3885
3886 // Warp 0 copies reduce element from transfer medium
3887 BasicBlock *W0ThenBB = BasicBlock::Create(Context&: Ctx, Name: "then");
3888 BasicBlock *W0ElseBB = BasicBlock::Create(Context&: Ctx, Name: "else");
3889 BasicBlock *W0MergeBB = BasicBlock::Create(Context&: Ctx, Name: "ifcont");
3890
3891 Value *NumWarpsVal =
3892 Builder.CreateLoad(Ty: Builder.getInt32Ty(), Ptr: NumWarpsAddrCast);
3893 // Up to 32 threads in warp 0 are active.
3894 Value *IsActiveThread =
3895 Builder.CreateICmpULT(LHS: GPUThreadID, RHS: NumWarpsVal, Name: "is_active_thread");
3896 Builder.CreateCondBr(Cond: IsActiveThread, True: W0ThenBB, False: W0ElseBB);
3897
3898 emitBlock(BB: W0ThenBB, CurFn: Builder.GetInsertBlock()->getParent());
3899
3900 // SecMediumPtr = &medium[tid]
3901 // SrcMediumVal = *SrcMediumPtr
3902 Value *SrcMediumPtrVal = Builder.CreateInBoundsGEP(
3903 Ty: ArrayTy, Ptr: TransferMedium, IdxList: {Builder.getInt64(C: 0), GPUThreadID});
3904 // TargetElemPtr = (CopyType*)(SrcDataAddr[i]) + I
3905 Value *TargetElemPtrPtr =
3906 Builder.CreateInBoundsGEP(Ty: RedListArrayTy, Ptr: ReduceList,
3907 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0),
3908 ConstantInt::get(Ty: IndexTy, V: En.index())});
3909 Value *TargetElemPtrVal =
3910 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: TargetElemPtrPtr);
3911 Value *TargetElemPtr = TargetElemPtrVal;
3912
3913 if (IsByRefElem && RI.DataPtrPtrGen) {
3914 InsertPointOrErrorTy GenRes =
3915 RI.DataPtrPtrGen(Builder.saveIP(), TargetElemPtr, TargetElemPtr);
3916
3917 if (!GenRes)
3918 return GenRes.takeError();
3919
3920 TargetElemPtr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: TargetElemPtr);
3921 }
3922
3923 if (NumIters > 1)
3924 TargetElemPtr =
3925 Builder.CreateGEP(Ty: Builder.getInt32Ty(), Ptr: TargetElemPtr, IdxList: Cnt);
3926
3927 // *TargetElemPtr = SrcMediumVal;
3928 Value *SrcMediumValue =
3929 Builder.CreateLoad(Ty: CType, Ptr: SrcMediumPtrVal, /*IsVolatile*/ isVolatile: true);
3930 Builder.CreateStore(Val: SrcMediumValue, Ptr: TargetElemPtr);
3931 Builder.CreateBr(Dest: W0MergeBB);
3932
3933 emitBlock(BB: W0ElseBB, CurFn: Builder.GetInsertBlock()->getParent());
3934 Builder.CreateBr(Dest: W0MergeBB);
3935
3936 emitBlock(BB: W0MergeBB, CurFn: Builder.GetInsertBlock()->getParent());
3937
3938 if (NumIters > 1) {
3939 Cnt = Builder.CreateNSWAdd(
3940 LHS: Cnt, RHS: ConstantInt::get(Ty: Builder.getInt32Ty(), /*V=*/1));
3941 Builder.CreateStore(Val: Cnt, Ptr: CntAddr, /*Volatile=*/isVolatile: false);
3942
3943 auto *CurFn = Builder.GetInsertBlock()->getParent();
3944 emitBranch(Target: PrecondBB);
3945 emitBlock(BB: ExitBB, CurFn);
3946 }
3947 RealTySize %= TySize;
3948 }
3949 }
3950
3951 Builder.CreateRetVoid();
3952
3953 return WcFunc;
3954}
3955
3956Expected<Function *> OpenMPIRBuilder::emitShuffleAndReduceFunction(
3957 ArrayRef<ReductionInfo> ReductionInfos, Function *ReduceFn,
3958 AttributeList FuncAttrs, ArrayRef<bool> IsByRef) {
3959 LLVMContext &Ctx = M.getContext();
3960 IRBuilder<>::InsertPointGuard IPG(Builder);
3961 FunctionType *FuncTy =
3962 FunctionType::get(Result: Builder.getVoidTy(),
3963 Params: {Builder.getPtrTy(), Builder.getInt16Ty(),
3964 Builder.getInt16Ty(), Builder.getInt16Ty()},
3965 /* IsVarArg */ isVarArg: false);
3966 Function *SarFunc =
3967 Function::Create(Ty: FuncTy, Linkage: GlobalVariable::InternalLinkage,
3968 N: "_omp_reduction_shuffle_and_reduce_func", M: &M);
3969 SarFunc->setCallingConv(Config.getRuntimeCC());
3970 SarFunc->setAttributes(FuncAttrs);
3971 SarFunc->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
3972 SarFunc->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
3973 SarFunc->addParamAttr(ArgNo: 2, Kind: Attribute::NoUndef);
3974 SarFunc->addParamAttr(ArgNo: 3, Kind: Attribute::NoUndef);
3975 SarFunc->addParamAttr(ArgNo: 1, Kind: Attribute::SExt);
3976 SarFunc->addParamAttr(ArgNo: 2, Kind: Attribute::SExt);
3977 SarFunc->addParamAttr(ArgNo: 3, Kind: Attribute::SExt);
3978 BasicBlock *EntryBB = BasicBlock::Create(Context&: M.getContext(), Name: "entry", Parent: SarFunc);
3979 Builder.SetInsertPoint(EntryBB);
3980 Builder.SetCurrentDebugLocation(llvm::DebugLoc());
3981
3982 // Thread local Reduce list used to host the values of data to be reduced.
3983 Argument *ReduceListArg = SarFunc->getArg(i: 0);
3984 // Current lane id; could be logical.
3985 Argument *LaneIDArg = SarFunc->getArg(i: 1);
3986 // Offset of the remote source lane relative to the current lane.
3987 Argument *RemoteLaneOffsetArg = SarFunc->getArg(i: 2);
3988 // Algorithm version. This is expected to be known at compile time.
3989 Argument *AlgoVerArg = SarFunc->getArg(i: 3);
3990
3991 Type *ReduceListArgType = ReduceListArg->getType();
3992 Type *LaneIDArgType = LaneIDArg->getType();
3993 Type *LaneIDArgPtrType = Builder.getPtrTy(AddrSpace: 0);
3994 Value *ReduceListAlloca = Builder.CreateAlloca(
3995 Ty: ReduceListArgType, ArraySize: nullptr, Name: ReduceListArg->getName() + ".addr");
3996 Value *LaneIdAlloca = Builder.CreateAlloca(Ty: LaneIDArgType, ArraySize: nullptr,
3997 Name: LaneIDArg->getName() + ".addr");
3998 Value *RemoteLaneOffsetAlloca = Builder.CreateAlloca(
3999 Ty: LaneIDArgType, ArraySize: nullptr, Name: RemoteLaneOffsetArg->getName() + ".addr");
4000 Value *AlgoVerAlloca = Builder.CreateAlloca(Ty: LaneIDArgType, ArraySize: nullptr,
4001 Name: AlgoVerArg->getName() + ".addr");
4002 ArrayType *RedListArrayTy =
4003 ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: ReductionInfos.size());
4004
4005 // Create a local thread-private variable to host the Reduce list
4006 // from a remote lane.
4007 Instruction *RemoteReductionListAlloca = Builder.CreateAlloca(
4008 Ty: RedListArrayTy, ArraySize: nullptr, Name: ".omp.reduction.remote_reduce_list");
4009
4010 Value *ReduceListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4011 V: ReduceListAlloca, DestTy: ReduceListArgType,
4012 Name: ReduceListAlloca->getName() + ".ascast");
4013 Value *LaneIdAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4014 V: LaneIdAlloca, DestTy: LaneIDArgPtrType, Name: LaneIdAlloca->getName() + ".ascast");
4015 Value *RemoteLaneOffsetAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4016 V: RemoteLaneOffsetAlloca, DestTy: LaneIDArgPtrType,
4017 Name: RemoteLaneOffsetAlloca->getName() + ".ascast");
4018 Value *AlgoVerAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4019 V: AlgoVerAlloca, DestTy: LaneIDArgPtrType, Name: AlgoVerAlloca->getName() + ".ascast");
4020 Value *RemoteListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4021 V: RemoteReductionListAlloca, DestTy: Builder.getPtrTy(),
4022 Name: RemoteReductionListAlloca->getName() + ".ascast");
4023
4024 Builder.CreateStore(Val: ReduceListArg, Ptr: ReduceListAddrCast);
4025 Builder.CreateStore(Val: LaneIDArg, Ptr: LaneIdAddrCast);
4026 Builder.CreateStore(Val: RemoteLaneOffsetArg, Ptr: RemoteLaneOffsetAddrCast);
4027 Builder.CreateStore(Val: AlgoVerArg, Ptr: AlgoVerAddrCast);
4028
4029 Value *ReduceList = Builder.CreateLoad(Ty: ReduceListArgType, Ptr: ReduceListAddrCast);
4030 Value *LaneId = Builder.CreateLoad(Ty: LaneIDArgType, Ptr: LaneIdAddrCast);
4031 Value *RemoteLaneOffset =
4032 Builder.CreateLoad(Ty: LaneIDArgType, Ptr: RemoteLaneOffsetAddrCast);
4033 Value *AlgoVer = Builder.CreateLoad(Ty: LaneIDArgType, Ptr: AlgoVerAddrCast);
4034
4035 InsertPointTy AllocaIP = getInsertPointAfterInstr(I: RemoteReductionListAlloca);
4036
4037 // This loop iterates through the list of reduce elements and copies,
4038 // element by element, from a remote lane in the warp to RemoteReduceList,
4039 // hosted on the thread's stack.
4040 Error EmitRedLsCpRes = emitReductionListCopy(
4041 AllocaIP, Action: CopyAction::RemoteLaneToThread, ReductionArrayTy: RedListArrayTy, ReductionInfos,
4042 SrcBase: ReduceList, DestBase: RemoteListAddrCast, IsByRef,
4043 CopyOptions: {.RemoteLaneOffset: RemoteLaneOffset, .ScratchpadIndex: nullptr, .ScratchpadWidth: nullptr});
4044
4045 if (EmitRedLsCpRes)
4046 return EmitRedLsCpRes;
4047
4048 // The actions to be performed on the Remote Reduce list is dependent
4049 // on the algorithm version.
4050 //
4051 // if (AlgoVer==0) || (AlgoVer==1 && (LaneId < Offset)) || (AlgoVer==2 &&
4052 // LaneId % 2 == 0 && Offset > 0):
4053 // do the reduction value aggregation
4054 //
4055 // The thread local variable Reduce list is mutated in place to host the
4056 // reduced data, which is the aggregated value produced from local and
4057 // remote lanes.
4058 //
4059 // Note that AlgoVer is expected to be a constant integer known at compile
4060 // time.
4061 // When AlgoVer==0, the first conjunction evaluates to true, making
4062 // the entire predicate true during compile time.
4063 // When AlgoVer==1, the second conjunction has only the second part to be
4064 // evaluated during runtime. Other conjunctions evaluates to false
4065 // during compile time.
4066 // When AlgoVer==2, the third conjunction has only the second part to be
4067 // evaluated during runtime. Other conjunctions evaluates to false
4068 // during compile time.
4069 Value *CondAlgo0 = Builder.CreateIsNull(Arg: AlgoVer);
4070 Value *Algo1 = Builder.CreateICmpEQ(LHS: AlgoVer, RHS: Builder.getInt16(C: 1));
4071 Value *LaneComp = Builder.CreateICmpULT(LHS: LaneId, RHS: RemoteLaneOffset);
4072 Value *CondAlgo1 = Builder.CreateAnd(LHS: Algo1, RHS: LaneComp);
4073 Value *Algo2 = Builder.CreateICmpEQ(LHS: AlgoVer, RHS: Builder.getInt16(C: 2));
4074 Value *LaneIdAnd1 = Builder.CreateAnd(LHS: LaneId, RHS: Builder.getInt16(C: 1));
4075 Value *LaneIdComp = Builder.CreateIsNull(Arg: LaneIdAnd1);
4076 Value *Algo2AndLaneIdComp = Builder.CreateAnd(LHS: Algo2, RHS: LaneIdComp);
4077 Value *RemoteOffsetComp =
4078 Builder.CreateICmpSGT(LHS: RemoteLaneOffset, RHS: Builder.getInt16(C: 0));
4079 Value *CondAlgo2 = Builder.CreateAnd(LHS: Algo2AndLaneIdComp, RHS: RemoteOffsetComp);
4080 Value *CA0OrCA1 = Builder.CreateOr(LHS: CondAlgo0, RHS: CondAlgo1);
4081 Value *CondReduce = Builder.CreateOr(LHS: CA0OrCA1, RHS: CondAlgo2);
4082
4083 BasicBlock *ThenBB = BasicBlock::Create(Context&: Ctx, Name: "then");
4084 BasicBlock *ElseBB = BasicBlock::Create(Context&: Ctx, Name: "else");
4085 BasicBlock *MergeBB = BasicBlock::Create(Context&: Ctx, Name: "ifcont");
4086
4087 Builder.CreateCondBr(Cond: CondReduce, True: ThenBB, False: ElseBB);
4088 emitBlock(BB: ThenBB, CurFn: Builder.GetInsertBlock()->getParent());
4089 Value *LocalReduceListPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
4090 V: ReduceList, DestTy: Builder.getPtrTy());
4091 Value *RemoteReduceListPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
4092 V: RemoteListAddrCast, DestTy: Builder.getPtrTy());
4093 createRuntimeFunctionCall(Callee: ReduceFn, Args: {LocalReduceListPtr, RemoteReduceListPtr})
4094 ->addFnAttr(Kind: Attribute::NoUnwind);
4095 Builder.CreateBr(Dest: MergeBB);
4096
4097 emitBlock(BB: ElseBB, CurFn: Builder.GetInsertBlock()->getParent());
4098 Builder.CreateBr(Dest: MergeBB);
4099
4100 emitBlock(BB: MergeBB, CurFn: Builder.GetInsertBlock()->getParent());
4101
4102 // if (AlgoVer==1 && (LaneId >= Offset)) copy Remote Reduce list to local
4103 // Reduce list.
4104 Algo1 = Builder.CreateICmpEQ(LHS: AlgoVer, RHS: Builder.getInt16(C: 1));
4105 Value *LaneIdGtOffset = Builder.CreateICmpUGE(LHS: LaneId, RHS: RemoteLaneOffset);
4106 Value *CondCopy = Builder.CreateAnd(LHS: Algo1, RHS: LaneIdGtOffset);
4107
4108 BasicBlock *CpyThenBB = BasicBlock::Create(Context&: Ctx, Name: "then");
4109 BasicBlock *CpyElseBB = BasicBlock::Create(Context&: Ctx, Name: "else");
4110 BasicBlock *CpyMergeBB = BasicBlock::Create(Context&: Ctx, Name: "ifcont");
4111 Builder.CreateCondBr(Cond: CondCopy, True: CpyThenBB, False: CpyElseBB);
4112
4113 emitBlock(BB: CpyThenBB, CurFn: Builder.GetInsertBlock()->getParent());
4114
4115 EmitRedLsCpRes = emitReductionListCopy(
4116 AllocaIP, Action: CopyAction::ThreadCopy, ReductionArrayTy: RedListArrayTy, ReductionInfos,
4117 SrcBase: RemoteListAddrCast, DestBase: ReduceList, IsByRef);
4118
4119 if (EmitRedLsCpRes)
4120 return EmitRedLsCpRes;
4121
4122 Builder.CreateBr(Dest: CpyMergeBB);
4123
4124 emitBlock(BB: CpyElseBB, CurFn: Builder.GetInsertBlock()->getParent());
4125 Builder.CreateBr(Dest: CpyMergeBB);
4126
4127 emitBlock(BB: CpyMergeBB, CurFn: Builder.GetInsertBlock()->getParent());
4128
4129 Builder.CreateRetVoid();
4130
4131 return SarFunc;
4132}
4133
4134OpenMPIRBuilder::InsertPointOrErrorTy
4135OpenMPIRBuilder::generateReductionDescriptor(
4136 Value *DescriptorAddr, Value *DataPtr, Value *SrcDescriptorAddr,
4137 Type *DescriptorType,
4138 function_ref<InsertPointOrErrorTy(InsertPointTy, Value *, Value *&)>
4139 DataPtrPtrGen) {
4140
4141 // Copy the source descriptor to preserve all metadata (rank, extents,
4142 // strides, etc.)
4143 Value *DescriptorSize =
4144 Builder.getInt64(C: M.getDataLayout().getTypeStoreSize(Ty: DescriptorType));
4145 Builder.CreateMemCpy(
4146 Dst: DescriptorAddr, DstAlign: M.getDataLayout().getPrefTypeAlign(Ty: DescriptorType),
4147 Src: SrcDescriptorAddr, SrcAlign: M.getDataLayout().getPrefTypeAlign(Ty: DescriptorType),
4148 Size: DescriptorSize);
4149
4150 // Update the base pointer field to point to the local shuffled data
4151 Value *DataPtrField;
4152 InsertPointOrErrorTy GenResult =
4153 DataPtrPtrGen(Builder.saveIP(), DescriptorAddr, DataPtrField);
4154
4155 if (!GenResult)
4156 return GenResult.takeError();
4157
4158 Builder.CreateStore(Val: Builder.CreatePointerBitCastOrAddrSpaceCast(
4159 V: DataPtr, DestTy: Builder.getPtrTy(), Name: ".ascast"),
4160 Ptr: DataPtrField);
4161
4162 return Builder.saveIP();
4163}
4164
4165Expected<Value *> OpenMPIRBuilder::createReductionDescriptorCopy(
4166 InsertPointTy AllocaIP, const ReductionInfo &RI, Value *DataPtr,
4167 Value *SrcDescriptorAddr, Type *DescriptorPtrTy, const Twine &Name) {
4168 InsertPointTy OldIP = Builder.saveIP();
4169 Builder.restoreIP(IP: AllocaIP);
4170
4171 AllocaInst *DescriptorAlloca =
4172 Builder.CreateAlloca(Ty: RI.ByRefAllocatedType, ArraySize: nullptr, Name);
4173 DescriptorAlloca->setAlignment(
4174 M.getDataLayout().getPrefTypeAlign(Ty: RI.ByRefAllocatedType));
4175 Value *DescriptorAddr = Builder.CreatePointerBitCastOrAddrSpaceCast(
4176 V: DescriptorAlloca, DestTy: DescriptorPtrTy,
4177 Name: DescriptorAlloca->getName() + ".ascast");
4178
4179 Builder.restoreIP(IP: OldIP);
4180
4181 InsertPointOrErrorTy GenResult =
4182 generateReductionDescriptor(DescriptorAddr, DataPtr, SrcDescriptorAddr,
4183 DescriptorType: RI.ByRefAllocatedType, DataPtrPtrGen: RI.DataPtrPtrGen);
4184 if (!GenResult)
4185 return GenResult.takeError();
4186
4187 return DescriptorAddr;
4188}
4189
4190Expected<Function *> OpenMPIRBuilder::emitListToGlobalCopyFunction(
4191 ArrayRef<ReductionInfo> ReductionInfos, Type *ReductionsBufferTy,
4192 AttributeList FuncAttrs, ArrayRef<bool> IsByRef) {
4193 IRBuilder<>::InsertPointGuard IPG(Builder);
4194 LLVMContext &Ctx = M.getContext();
4195 FunctionType *FuncTy = FunctionType::get(
4196 Result: Builder.getVoidTy(),
4197 Params: {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},
4198 /* IsVarArg */ isVarArg: false);
4199 Function *LtGCFunc =
4200 Function::Create(Ty: FuncTy, Linkage: GlobalVariable::InternalLinkage,
4201 N: "_omp_reduction_list_to_global_copy_func", M: &M);
4202 LtGCFunc->setAttributes(FuncAttrs);
4203 LtGCFunc->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
4204 LtGCFunc->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
4205 LtGCFunc->addParamAttr(ArgNo: 2, Kind: Attribute::NoUndef);
4206
4207 BasicBlock *EntryBlock = BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: LtGCFunc);
4208 Builder.SetInsertPoint(EntryBlock);
4209 Builder.SetCurrentDebugLocation(llvm::DebugLoc());
4210
4211 // Buffer: global reduction buffer.
4212 Argument *BufferArg = LtGCFunc->getArg(i: 0);
4213 // Idx: index of the buffer.
4214 Argument *IdxArg = LtGCFunc->getArg(i: 1);
4215 // ReduceList: thread local Reduce list.
4216 Argument *ReduceListArg = LtGCFunc->getArg(i: 2);
4217
4218 Value *BufferArgAlloca = Builder.CreateAlloca(Ty: Builder.getPtrTy(), ArraySize: nullptr,
4219 Name: BufferArg->getName() + ".addr");
4220 Value *IdxArgAlloca = Builder.CreateAlloca(Ty: Builder.getInt32Ty(), ArraySize: nullptr,
4221 Name: IdxArg->getName() + ".addr");
4222 Value *ReduceListArgAlloca = Builder.CreateAlloca(
4223 Ty: Builder.getPtrTy(), ArraySize: nullptr, Name: ReduceListArg->getName() + ".addr");
4224 Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4225 V: BufferArgAlloca, DestTy: Builder.getPtrTy(),
4226 Name: BufferArgAlloca->getName() + ".ascast");
4227 Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4228 V: IdxArgAlloca, DestTy: Builder.getPtrTy(), Name: IdxArgAlloca->getName() + ".ascast");
4229 Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4230 V: ReduceListArgAlloca, DestTy: Builder.getPtrTy(),
4231 Name: ReduceListArgAlloca->getName() + ".ascast");
4232
4233 Builder.CreateStore(Val: BufferArg, Ptr: BufferArgAddrCast);
4234 Builder.CreateStore(Val: IdxArg, Ptr: IdxArgAddrCast);
4235 Builder.CreateStore(Val: ReduceListArg, Ptr: ReduceListArgAddrCast);
4236
4237 Value *LocalReduceList =
4238 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ReduceListArgAddrCast);
4239 Value *BufferArgVal =
4240 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: BufferArgAddrCast);
4241 Value *Idxs[] = {Builder.CreateLoad(Ty: Builder.getInt32Ty(), Ptr: IdxArgAddrCast)};
4242 Type *IndexTy = Builder.getIndexTy(
4243 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
4244 for (auto En : enumerate(First&: ReductionInfos)) {
4245 const ReductionInfo &RI = En.value();
4246 auto *RedListArrayTy =
4247 ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: ReductionInfos.size());
4248 // Reduce element = LocalReduceList[i]
4249 Value *ElemPtrPtr = Builder.CreateInBoundsGEP(
4250 Ty: RedListArrayTy, Ptr: LocalReduceList,
4251 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
4252 // elemptr = ((CopyType*)(elemptrptr)) + I
4253 Value *ElemPtr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ElemPtrPtr);
4254
4255 // Global = Buffer.VD[Idx];
4256 Value *BufferVD =
4257 Builder.CreateInBoundsGEP(Ty: ReductionsBufferTy, Ptr: BufferArgVal, IdxList: Idxs);
4258 Value *GlobVal = Builder.CreateConstInBoundsGEP2_32(
4259 Ty: ReductionsBufferTy, Ptr: BufferVD, Idx0: 0, Idx1: En.index());
4260
4261 switch (RI.EvaluationKind) {
4262 case EvalKind::Scalar: {
4263 Value *TargetElement;
4264
4265 if (IsByRef.empty() || !IsByRef[En.index()]) {
4266 TargetElement = Builder.CreateLoad(Ty: RI.ElementType, Ptr: ElemPtr);
4267 } else {
4268 if (RI.DataPtrPtrGen) {
4269 InsertPointOrErrorTy GenResult =
4270 RI.DataPtrPtrGen(Builder.saveIP(), ElemPtr, ElemPtr);
4271
4272 if (!GenResult)
4273 return GenResult.takeError();
4274
4275 ElemPtr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ElemPtr);
4276 }
4277 TargetElement = Builder.CreateLoad(Ty: RI.ByRefElementType, Ptr: ElemPtr);
4278 }
4279
4280 Builder.CreateStore(Val: TargetElement, Ptr: GlobVal);
4281 break;
4282 }
4283 case EvalKind::Complex: {
4284 Value *SrcRealPtr = Builder.CreateConstInBoundsGEP2_32(
4285 Ty: RI.ElementType, Ptr: ElemPtr, Idx0: 0, Idx1: 0, Name: ".realp");
4286 Value *SrcReal = Builder.CreateLoad(
4287 Ty: RI.ElementType->getStructElementType(N: 0), Ptr: SrcRealPtr, Name: ".real");
4288 Value *SrcImgPtr = Builder.CreateConstInBoundsGEP2_32(
4289 Ty: RI.ElementType, Ptr: ElemPtr, Idx0: 0, Idx1: 1, Name: ".imagp");
4290 Value *SrcImg = Builder.CreateLoad(
4291 Ty: RI.ElementType->getStructElementType(N: 1), Ptr: SrcImgPtr, Name: ".imag");
4292
4293 Value *DestRealPtr = Builder.CreateConstInBoundsGEP2_32(
4294 Ty: RI.ElementType, Ptr: GlobVal, Idx0: 0, Idx1: 0, Name: ".realp");
4295 Value *DestImgPtr = Builder.CreateConstInBoundsGEP2_32(
4296 Ty: RI.ElementType, Ptr: GlobVal, Idx0: 0, Idx1: 1, Name: ".imagp");
4297 Builder.CreateStore(Val: SrcReal, Ptr: DestRealPtr);
4298 Builder.CreateStore(Val: SrcImg, Ptr: DestImgPtr);
4299 break;
4300 }
4301 case EvalKind::Aggregate: {
4302 Value *SizeVal =
4303 Builder.getInt64(C: M.getDataLayout().getTypeStoreSize(Ty: RI.ElementType));
4304 Builder.CreateMemCpy(
4305 Dst: GlobVal, DstAlign: M.getDataLayout().getPrefTypeAlign(Ty: RI.ElementType), Src: ElemPtr,
4306 SrcAlign: M.getDataLayout().getPrefTypeAlign(Ty: RI.ElementType), Size: SizeVal, isVolatile: false);
4307 break;
4308 }
4309 }
4310 }
4311
4312 Builder.CreateRetVoid();
4313 return LtGCFunc;
4314}
4315
4316Expected<Function *> OpenMPIRBuilder::emitListToGlobalReduceFunction(
4317 ArrayRef<ReductionInfo> ReductionInfos, Function *ReduceFn,
4318 Type *ReductionsBufferTy, AttributeList FuncAttrs, ArrayRef<bool> IsByRef) {
4319 IRBuilder<>::InsertPointGuard IPG(Builder);
4320 LLVMContext &Ctx = M.getContext();
4321 FunctionType *FuncTy = FunctionType::get(
4322 Result: Builder.getVoidTy(),
4323 Params: {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},
4324 /* IsVarArg */ isVarArg: false);
4325 Function *LtGRFunc =
4326 Function::Create(Ty: FuncTy, Linkage: GlobalVariable::InternalLinkage,
4327 N: "_omp_reduction_list_to_global_reduce_func", M: &M);
4328 LtGRFunc->setAttributes(FuncAttrs);
4329 LtGRFunc->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
4330 LtGRFunc->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
4331 LtGRFunc->addParamAttr(ArgNo: 2, Kind: Attribute::NoUndef);
4332
4333 BasicBlock *EntryBlock = BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: LtGRFunc);
4334 Builder.SetInsertPoint(EntryBlock);
4335 Builder.SetCurrentDebugLocation(llvm::DebugLoc());
4336
4337 // Buffer: global reduction buffer.
4338 Argument *BufferArg = LtGRFunc->getArg(i: 0);
4339 // Idx: index of the buffer.
4340 Argument *IdxArg = LtGRFunc->getArg(i: 1);
4341 // ReduceList: thread local Reduce list.
4342 Argument *ReduceListArg = LtGRFunc->getArg(i: 2);
4343
4344 Value *BufferArgAlloca = Builder.CreateAlloca(Ty: Builder.getPtrTy(), ArraySize: nullptr,
4345 Name: BufferArg->getName() + ".addr");
4346 Value *IdxArgAlloca = Builder.CreateAlloca(Ty: Builder.getInt32Ty(), ArraySize: nullptr,
4347 Name: IdxArg->getName() + ".addr");
4348 Value *ReduceListArgAlloca = Builder.CreateAlloca(
4349 Ty: Builder.getPtrTy(), ArraySize: nullptr, Name: ReduceListArg->getName() + ".addr");
4350 auto *RedListArrayTy =
4351 ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: ReductionInfos.size());
4352
4353 // 1. Build a list of reduction variables.
4354 // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
4355 Value *LocalReduceList =
4356 Builder.CreateAlloca(Ty: RedListArrayTy, ArraySize: nullptr, Name: ".omp.reduction.red_list");
4357
4358 InsertPointTy AllocaIP{EntryBlock, EntryBlock->begin()};
4359
4360 Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4361 V: BufferArgAlloca, DestTy: Builder.getPtrTy(),
4362 Name: BufferArgAlloca->getName() + ".ascast");
4363 Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4364 V: IdxArgAlloca, DestTy: Builder.getPtrTy(), Name: IdxArgAlloca->getName() + ".ascast");
4365 Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4366 V: ReduceListArgAlloca, DestTy: Builder.getPtrTy(),
4367 Name: ReduceListArgAlloca->getName() + ".ascast");
4368 Value *LocalReduceListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4369 V: LocalReduceList, DestTy: Builder.getPtrTy(),
4370 Name: LocalReduceList->getName() + ".ascast");
4371
4372 Builder.CreateStore(Val: BufferArg, Ptr: BufferArgAddrCast);
4373 Builder.CreateStore(Val: IdxArg, Ptr: IdxArgAddrCast);
4374 Builder.CreateStore(Val: ReduceListArg, Ptr: ReduceListArgAddrCast);
4375
4376 Value *BufferVal = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: BufferArgAddrCast);
4377 Value *Idxs[] = {Builder.CreateLoad(Ty: Builder.getInt32Ty(), Ptr: IdxArgAddrCast)};
4378 Type *IndexTy = Builder.getIndexTy(
4379 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
4380 for (auto En : enumerate(First&: ReductionInfos)) {
4381 const ReductionInfo &RI = En.value();
4382
4383 Value *TargetElementPtrPtr = Builder.CreateInBoundsGEP(
4384 Ty: RedListArrayTy, Ptr: LocalReduceListAddrCast,
4385 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
4386 Value *BufferVD =
4387 Builder.CreateInBoundsGEP(Ty: ReductionsBufferTy, Ptr: BufferVal, IdxList: Idxs);
4388 // Global = Buffer.VD[Idx];
4389 Value *GlobValPtr = Builder.CreateConstInBoundsGEP2_32(
4390 Ty: ReductionsBufferTy, Ptr: BufferVD, Idx0: 0, Idx1: En.index());
4391
4392 if (!IsByRef.empty() && IsByRef[En.index()] && RI.DataPtrPtrGen) {
4393 // Get source descriptor from the reduce list argument
4394 Value *ReduceList =
4395 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ReduceListArgAddrCast);
4396 Value *SrcElementPtrPtr =
4397 Builder.CreateInBoundsGEP(Ty: RedListArrayTy, Ptr: ReduceList,
4398 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0),
4399 ConstantInt::get(Ty: IndexTy, V: En.index())});
4400 Value *SrcDescriptorAddr =
4401 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: SrcElementPtrPtr);
4402
4403 // Copy descriptor from source and update base_ptr to global buffer data
4404 Expected<Value *> ByRefAlloc = createReductionDescriptorCopy(
4405 AllocaIP, RI, DataPtr: GlobValPtr, SrcDescriptorAddr, DescriptorPtrTy: Builder.getPtrTy());
4406 if (!ByRefAlloc)
4407 return ByRefAlloc.takeError();
4408
4409 Builder.CreateStore(Val: *ByRefAlloc, Ptr: TargetElementPtrPtr);
4410 } else {
4411 Builder.CreateStore(Val: GlobValPtr, Ptr: TargetElementPtrPtr);
4412 }
4413 }
4414
4415 // Call reduce_function(GlobalReduceList, ReduceList)
4416 Value *ReduceList =
4417 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ReduceListArgAddrCast);
4418 createRuntimeFunctionCall(Callee: ReduceFn, Args: {LocalReduceListAddrCast, ReduceList})
4419 ->addFnAttr(Kind: Attribute::NoUnwind);
4420 Builder.CreateRetVoid();
4421 return LtGRFunc;
4422}
4423
4424Expected<Function *> OpenMPIRBuilder::emitGlobalToListCopyFunction(
4425 ArrayRef<ReductionInfo> ReductionInfos, Type *ReductionsBufferTy,
4426 AttributeList FuncAttrs, ArrayRef<bool> IsByRef) {
4427 IRBuilder<>::InsertPointGuard IPG(Builder);
4428 LLVMContext &Ctx = M.getContext();
4429 FunctionType *FuncTy = FunctionType::get(
4430 Result: Builder.getVoidTy(),
4431 Params: {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},
4432 /* IsVarArg */ isVarArg: false);
4433 Function *GtLCFunc =
4434 Function::Create(Ty: FuncTy, Linkage: GlobalVariable::InternalLinkage,
4435 N: "_omp_reduction_global_to_list_copy_func", M: &M);
4436 GtLCFunc->setAttributes(FuncAttrs);
4437 GtLCFunc->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
4438 GtLCFunc->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
4439 GtLCFunc->addParamAttr(ArgNo: 2, Kind: Attribute::NoUndef);
4440
4441 BasicBlock *EntryBlock = BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: GtLCFunc);
4442 Builder.SetInsertPoint(EntryBlock);
4443 Builder.SetCurrentDebugLocation(llvm::DebugLoc());
4444
4445 // Buffer: global reduction buffer.
4446 Argument *BufferArg = GtLCFunc->getArg(i: 0);
4447 // Idx: index of the buffer.
4448 Argument *IdxArg = GtLCFunc->getArg(i: 1);
4449 // ReduceList: thread local Reduce list.
4450 Argument *ReduceListArg = GtLCFunc->getArg(i: 2);
4451
4452 Value *BufferArgAlloca = Builder.CreateAlloca(Ty: Builder.getPtrTy(), ArraySize: nullptr,
4453 Name: BufferArg->getName() + ".addr");
4454 Value *IdxArgAlloca = Builder.CreateAlloca(Ty: Builder.getInt32Ty(), ArraySize: nullptr,
4455 Name: IdxArg->getName() + ".addr");
4456 Value *ReduceListArgAlloca = Builder.CreateAlloca(
4457 Ty: Builder.getPtrTy(), ArraySize: nullptr, Name: ReduceListArg->getName() + ".addr");
4458 Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4459 V: BufferArgAlloca, DestTy: Builder.getPtrTy(),
4460 Name: BufferArgAlloca->getName() + ".ascast");
4461 Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4462 V: IdxArgAlloca, DestTy: Builder.getPtrTy(), Name: IdxArgAlloca->getName() + ".ascast");
4463 Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4464 V: ReduceListArgAlloca, DestTy: Builder.getPtrTy(),
4465 Name: ReduceListArgAlloca->getName() + ".ascast");
4466 Builder.CreateStore(Val: BufferArg, Ptr: BufferArgAddrCast);
4467 Builder.CreateStore(Val: IdxArg, Ptr: IdxArgAddrCast);
4468 Builder.CreateStore(Val: ReduceListArg, Ptr: ReduceListArgAddrCast);
4469
4470 Value *LocalReduceList =
4471 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ReduceListArgAddrCast);
4472 Value *BufferVal = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: BufferArgAddrCast);
4473 Value *Idxs[] = {Builder.CreateLoad(Ty: Builder.getInt32Ty(), Ptr: IdxArgAddrCast)};
4474 Type *IndexTy = Builder.getIndexTy(
4475 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
4476 for (auto En : enumerate(First&: ReductionInfos)) {
4477 const OpenMPIRBuilder::ReductionInfo &RI = En.value();
4478 auto *RedListArrayTy =
4479 ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: ReductionInfos.size());
4480 // Reduce element = LocalReduceList[i]
4481 Value *ElemPtrPtr = Builder.CreateInBoundsGEP(
4482 Ty: RedListArrayTy, Ptr: LocalReduceList,
4483 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
4484 // elemptr = ((CopyType*)(elemptrptr)) + I
4485 Value *ElemPtr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ElemPtrPtr);
4486 // Global = Buffer.VD[Idx];
4487 Value *BufferVD =
4488 Builder.CreateInBoundsGEP(Ty: ReductionsBufferTy, Ptr: BufferVal, IdxList: Idxs);
4489 Value *GlobValPtr = Builder.CreateConstInBoundsGEP2_32(
4490 Ty: ReductionsBufferTy, Ptr: BufferVD, Idx0: 0, Idx1: En.index());
4491
4492 switch (RI.EvaluationKind) {
4493 case EvalKind::Scalar: {
4494 Type *ElemType = RI.ElementType;
4495
4496 if (!IsByRef.empty() && IsByRef[En.index()]) {
4497 ElemType = RI.ByRefElementType;
4498 if (RI.DataPtrPtrGen) {
4499 InsertPointOrErrorTy GenResult =
4500 RI.DataPtrPtrGen(Builder.saveIP(), ElemPtr, ElemPtr);
4501
4502 if (!GenResult)
4503 return GenResult.takeError();
4504
4505 ElemPtr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ElemPtr);
4506 }
4507 }
4508
4509 Value *TargetElement = Builder.CreateLoad(Ty: ElemType, Ptr: GlobValPtr);
4510 Builder.CreateStore(Val: TargetElement, Ptr: ElemPtr);
4511 break;
4512 }
4513 case EvalKind::Complex: {
4514 Value *SrcRealPtr = Builder.CreateConstInBoundsGEP2_32(
4515 Ty: RI.ElementType, Ptr: GlobValPtr, Idx0: 0, Idx1: 0, Name: ".realp");
4516 Value *SrcReal = Builder.CreateLoad(
4517 Ty: RI.ElementType->getStructElementType(N: 0), Ptr: SrcRealPtr, Name: ".real");
4518 Value *SrcImgPtr = Builder.CreateConstInBoundsGEP2_32(
4519 Ty: RI.ElementType, Ptr: GlobValPtr, Idx0: 0, Idx1: 1, Name: ".imagp");
4520 Value *SrcImg = Builder.CreateLoad(
4521 Ty: RI.ElementType->getStructElementType(N: 1), Ptr: SrcImgPtr, Name: ".imag");
4522
4523 Value *DestRealPtr = Builder.CreateConstInBoundsGEP2_32(
4524 Ty: RI.ElementType, Ptr: ElemPtr, Idx0: 0, Idx1: 0, Name: ".realp");
4525 Value *DestImgPtr = Builder.CreateConstInBoundsGEP2_32(
4526 Ty: RI.ElementType, Ptr: ElemPtr, Idx0: 0, Idx1: 1, Name: ".imagp");
4527 Builder.CreateStore(Val: SrcReal, Ptr: DestRealPtr);
4528 Builder.CreateStore(Val: SrcImg, Ptr: DestImgPtr);
4529 break;
4530 }
4531 case EvalKind::Aggregate: {
4532 Value *SizeVal =
4533 Builder.getInt64(C: M.getDataLayout().getTypeStoreSize(Ty: RI.ElementType));
4534 Builder.CreateMemCpy(
4535 Dst: ElemPtr, DstAlign: M.getDataLayout().getPrefTypeAlign(Ty: RI.ElementType),
4536 Src: GlobValPtr, SrcAlign: M.getDataLayout().getPrefTypeAlign(Ty: RI.ElementType),
4537 Size: SizeVal, isVolatile: false);
4538 break;
4539 }
4540 }
4541 }
4542
4543 Builder.CreateRetVoid();
4544 return GtLCFunc;
4545}
4546
4547Expected<Function *> OpenMPIRBuilder::emitGlobalToListReduceFunction(
4548 ArrayRef<ReductionInfo> ReductionInfos, Function *ReduceFn,
4549 Type *ReductionsBufferTy, AttributeList FuncAttrs, ArrayRef<bool> IsByRef) {
4550 IRBuilder<>::InsertPointGuard IPG(Builder);
4551 LLVMContext &Ctx = M.getContext();
4552 auto *FuncTy = FunctionType::get(
4553 Result: Builder.getVoidTy(),
4554 Params: {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},
4555 /* IsVarArg */ isVarArg: false);
4556 Function *GtLRFunc =
4557 Function::Create(Ty: FuncTy, Linkage: GlobalVariable::InternalLinkage,
4558 N: "_omp_reduction_global_to_list_reduce_func", M: &M);
4559 GtLRFunc->setAttributes(FuncAttrs);
4560 GtLRFunc->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
4561 GtLRFunc->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
4562 GtLRFunc->addParamAttr(ArgNo: 2, Kind: Attribute::NoUndef);
4563
4564 BasicBlock *EntryBlock = BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: GtLRFunc);
4565 Builder.SetInsertPoint(EntryBlock);
4566 Builder.SetCurrentDebugLocation(llvm::DebugLoc());
4567
4568 // Buffer: global reduction buffer.
4569 Argument *BufferArg = GtLRFunc->getArg(i: 0);
4570 // Idx: index of the buffer.
4571 Argument *IdxArg = GtLRFunc->getArg(i: 1);
4572 // ReduceList: thread local Reduce list.
4573 Argument *ReduceListArg = GtLRFunc->getArg(i: 2);
4574
4575 Value *BufferArgAlloca = Builder.CreateAlloca(Ty: Builder.getPtrTy(), ArraySize: nullptr,
4576 Name: BufferArg->getName() + ".addr");
4577 Value *IdxArgAlloca = Builder.CreateAlloca(Ty: Builder.getInt32Ty(), ArraySize: nullptr,
4578 Name: IdxArg->getName() + ".addr");
4579 Value *ReduceListArgAlloca = Builder.CreateAlloca(
4580 Ty: Builder.getPtrTy(), ArraySize: nullptr, Name: ReduceListArg->getName() + ".addr");
4581 ArrayType *RedListArrayTy =
4582 ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: ReductionInfos.size());
4583
4584 // 1. Build a list of reduction variables.
4585 // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
4586 Value *LocalReduceList =
4587 Builder.CreateAlloca(Ty: RedListArrayTy, ArraySize: nullptr, Name: ".omp.reduction.red_list");
4588
4589 InsertPointTy AllocaIP{EntryBlock, EntryBlock->begin()};
4590
4591 Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4592 V: BufferArgAlloca, DestTy: Builder.getPtrTy(),
4593 Name: BufferArgAlloca->getName() + ".ascast");
4594 Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4595 V: IdxArgAlloca, DestTy: Builder.getPtrTy(), Name: IdxArgAlloca->getName() + ".ascast");
4596 Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4597 V: ReduceListArgAlloca, DestTy: Builder.getPtrTy(),
4598 Name: ReduceListArgAlloca->getName() + ".ascast");
4599 Value *ReductionList = Builder.CreatePointerBitCastOrAddrSpaceCast(
4600 V: LocalReduceList, DestTy: Builder.getPtrTy(),
4601 Name: LocalReduceList->getName() + ".ascast");
4602
4603 Builder.CreateStore(Val: BufferArg, Ptr: BufferArgAddrCast);
4604 Builder.CreateStore(Val: IdxArg, Ptr: IdxArgAddrCast);
4605 Builder.CreateStore(Val: ReduceListArg, Ptr: ReduceListArgAddrCast);
4606
4607 Value *BufferVal = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: BufferArgAddrCast);
4608 Value *Idxs[] = {Builder.CreateLoad(Ty: Builder.getInt32Ty(), Ptr: IdxArgAddrCast)};
4609 Type *IndexTy = Builder.getIndexTy(
4610 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
4611 for (auto En : enumerate(First&: ReductionInfos)) {
4612 const ReductionInfo &RI = En.value();
4613
4614 Value *TargetElementPtrPtr = Builder.CreateInBoundsGEP(
4615 Ty: RedListArrayTy, Ptr: ReductionList,
4616 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
4617 // Global = Buffer.VD[Idx];
4618 Value *BufferVD =
4619 Builder.CreateInBoundsGEP(Ty: ReductionsBufferTy, Ptr: BufferVal, IdxList: Idxs);
4620 Value *GlobValPtr = Builder.CreateConstInBoundsGEP2_32(
4621 Ty: ReductionsBufferTy, Ptr: BufferVD, Idx0: 0, Idx1: En.index());
4622
4623 if (!IsByRef.empty() && IsByRef[En.index()] && RI.DataPtrPtrGen) {
4624 // Get source descriptor from the reduce list
4625 Value *ReduceListVal =
4626 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ReduceListArgAddrCast);
4627 Value *SrcElementPtrPtr =
4628 Builder.CreateInBoundsGEP(Ty: RedListArrayTy, Ptr: ReduceListVal,
4629 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0),
4630 ConstantInt::get(Ty: IndexTy, V: En.index())});
4631 Value *SrcDescriptorAddr =
4632 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: SrcElementPtrPtr);
4633
4634 // Copy descriptor from source and update base_ptr to global buffer data
4635 Expected<Value *> ByRefAlloc = createReductionDescriptorCopy(
4636 AllocaIP, RI, DataPtr: GlobValPtr, SrcDescriptorAddr, DescriptorPtrTy: Builder.getPtrTy());
4637 if (!ByRefAlloc)
4638 return ByRefAlloc.takeError();
4639
4640 Builder.CreateStore(Val: *ByRefAlloc, Ptr: TargetElementPtrPtr);
4641 } else {
4642 Builder.CreateStore(Val: GlobValPtr, Ptr: TargetElementPtrPtr);
4643 }
4644 }
4645
4646 // Call reduce_function(ReduceList, GlobalReduceList)
4647 Value *ReduceList =
4648 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ReduceListArgAddrCast);
4649 createRuntimeFunctionCall(Callee: ReduceFn, Args: {ReduceList, ReductionList})
4650 ->addFnAttr(Kind: Attribute::NoUnwind);
4651 Builder.CreateRetVoid();
4652 return GtLRFunc;
4653}
4654
4655std::string OpenMPIRBuilder::getReductionFuncName(StringRef Name) const {
4656 std::string Suffix =
4657 createPlatformSpecificName(Parts: {"omp", "reduction", "reduction_func"});
4658 return (Name + Suffix).str();
4659}
4660
4661Expected<Function *> OpenMPIRBuilder::createReductionFunction(
4662 StringRef ReducerName, ArrayRef<ReductionInfo> ReductionInfos,
4663 ArrayRef<bool> IsByRef, ReductionGenCBKind ReductionGenCBKind,
4664 AttributeList FuncAttrs) {
4665 IRBuilder<>::InsertPointGuard IPG(Builder);
4666 auto *FuncTy = FunctionType::get(Result: Builder.getVoidTy(),
4667 Params: {Builder.getPtrTy(), Builder.getPtrTy()},
4668 /* IsVarArg */ isVarArg: false);
4669 std::string Name = getReductionFuncName(Name: ReducerName);
4670 Function *ReductionFunc =
4671 Function::Create(Ty: FuncTy, Linkage: GlobalVariable::InternalLinkage, N: Name, M: &M);
4672 ReductionFunc->setCallingConv(Config.getRuntimeCC());
4673 ReductionFunc->setAttributes(FuncAttrs);
4674 ReductionFunc->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
4675 ReductionFunc->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
4676 BasicBlock *EntryBB =
4677 BasicBlock::Create(Context&: M.getContext(), Name: "entry", Parent: ReductionFunc);
4678 Builder.SetInsertPoint(EntryBB);
4679 Builder.SetCurrentDebugLocation(llvm::DebugLoc());
4680
4681 // Need to alloca memory here and deal with the pointers before getting
4682 // LHS/RHS pointers out
4683 Value *LHSArrayPtr = nullptr;
4684 Value *RHSArrayPtr = nullptr;
4685 Argument *Arg0 = ReductionFunc->getArg(i: 0);
4686 Argument *Arg1 = ReductionFunc->getArg(i: 1);
4687 Type *Arg0Type = Arg0->getType();
4688 Type *Arg1Type = Arg1->getType();
4689
4690 Value *LHSAlloca =
4691 Builder.CreateAlloca(Ty: Arg0Type, ArraySize: nullptr, Name: Arg0->getName() + ".addr");
4692 Value *RHSAlloca =
4693 Builder.CreateAlloca(Ty: Arg1Type, ArraySize: nullptr, Name: Arg1->getName() + ".addr");
4694 Value *LHSAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4695 V: LHSAlloca, DestTy: Arg0Type, Name: LHSAlloca->getName() + ".ascast");
4696 Value *RHSAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4697 V: RHSAlloca, DestTy: Arg1Type, Name: RHSAlloca->getName() + ".ascast");
4698 Builder.CreateStore(Val: Arg0, Ptr: LHSAddrCast);
4699 Builder.CreateStore(Val: Arg1, Ptr: RHSAddrCast);
4700 LHSArrayPtr = Builder.CreateLoad(Ty: Arg0Type, Ptr: LHSAddrCast);
4701 RHSArrayPtr = Builder.CreateLoad(Ty: Arg1Type, Ptr: RHSAddrCast);
4702
4703 Type *RedArrayTy = ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: ReductionInfos.size());
4704 Type *IndexTy = Builder.getIndexTy(
4705 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
4706 SmallVector<Value *> LHSPtrs, RHSPtrs;
4707 for (auto En : enumerate(First&: ReductionInfos)) {
4708 const ReductionInfo &RI = En.value();
4709 Value *RHSI8PtrPtr = Builder.CreateInBoundsGEP(
4710 Ty: RedArrayTy, Ptr: RHSArrayPtr,
4711 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
4712 Value *RHSI8Ptr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: RHSI8PtrPtr);
4713 Value *RHSPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
4714 V: RHSI8Ptr, DestTy: RI.PrivateVariable->getType(),
4715 Name: RHSI8Ptr->getName() + ".ascast");
4716
4717 Value *LHSI8PtrPtr = Builder.CreateInBoundsGEP(
4718 Ty: RedArrayTy, Ptr: LHSArrayPtr,
4719 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
4720 Value *LHSI8Ptr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: LHSI8PtrPtr);
4721 Value *LHSPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
4722 V: LHSI8Ptr, DestTy: RI.Variable->getType(), Name: LHSI8Ptr->getName() + ".ascast");
4723
4724 if (ReductionGenCBKind == ReductionGenCBKind::Clang) {
4725 LHSPtrs.emplace_back(Args&: LHSPtr);
4726 RHSPtrs.emplace_back(Args&: RHSPtr);
4727 } else {
4728 Value *LHS = LHSPtr;
4729 Value *RHS = RHSPtr;
4730
4731 if (!IsByRef.empty() && !IsByRef[En.index()]) {
4732 LHS = Builder.CreateLoad(Ty: RI.ElementType, Ptr: LHSPtr);
4733 RHS = Builder.CreateLoad(Ty: RI.ElementType, Ptr: RHSPtr);
4734 }
4735
4736 Value *Reduced;
4737 InsertPointOrErrorTy AfterIP =
4738 RI.ReductionGen(Builder.saveIP(), LHS, RHS, Reduced);
4739 if (!AfterIP)
4740 return AfterIP.takeError();
4741 if (!Builder.GetInsertBlock())
4742 return ReductionFunc;
4743
4744 Builder.restoreIP(IP: *AfterIP);
4745
4746 if (!IsByRef.empty() && !IsByRef[En.index()])
4747 Builder.CreateStore(Val: Reduced, Ptr: LHSPtr);
4748 }
4749 }
4750
4751 if (ReductionGenCBKind == ReductionGenCBKind::Clang)
4752 for (auto En : enumerate(First&: ReductionInfos)) {
4753 unsigned Index = En.index();
4754 const ReductionInfo &RI = En.value();
4755 Value *LHSFixupPtr, *RHSFixupPtr;
4756 Builder.restoreIP(IP: RI.ReductionGenClang(
4757 Builder.saveIP(), Index, &LHSFixupPtr, &RHSFixupPtr, ReductionFunc));
4758
4759 // Fix the CallBack code genereated to use the correct Values for the LHS
4760 // and RHS
4761 LHSFixupPtr->replaceUsesWithIf(
4762 New: LHSPtrs[Index], ShouldReplace: [ReductionFunc](const Use &U) {
4763 return cast<Instruction>(Val: U.getUser())->getParent()->getParent() ==
4764 ReductionFunc;
4765 });
4766 RHSFixupPtr->replaceUsesWithIf(
4767 New: RHSPtrs[Index], ShouldReplace: [ReductionFunc](const Use &U) {
4768 return cast<Instruction>(Val: U.getUser())->getParent()->getParent() ==
4769 ReductionFunc;
4770 });
4771 }
4772
4773 Builder.CreateRetVoid();
4774 // Compiling with `-O0`, `alloca`s emitted in non-entry blocks are not hoisted
4775 // to the entry block (this is dones for higher opt levels by later passes in
4776 // the pipeline). This has caused issues because non-entry `alloca`s force the
4777 // function to use dynamic stack allocations and we might run out of scratch
4778 // memory.
4779 hoistNonEntryAllocasToEntryBlock(Func: ReductionFunc);
4780
4781 return ReductionFunc;
4782}
4783
4784static void
4785checkReductionInfos(ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos,
4786 bool IsGPU) {
4787 for (const OpenMPIRBuilder::ReductionInfo &RI : ReductionInfos) {
4788 (void)RI;
4789 assert(RI.Variable && "expected non-null variable");
4790 assert(RI.PrivateVariable && "expected non-null private variable");
4791 assert((RI.ReductionGen || RI.ReductionGenClang) &&
4792 "expected non-null reduction generator callback");
4793 if (!IsGPU) {
4794 assert(
4795 RI.Variable->getType() == RI.PrivateVariable->getType() &&
4796 "expected variables and their private equivalents to have the same "
4797 "type");
4798 }
4799 assert(RI.Variable->getType()->isPointerTy() &&
4800 "expected variables to be pointers");
4801 }
4802}
4803
4804// The atomic cross-team reduction fast path applies when every reduction in the
4805// set can be represented by an atomicrmw. Clang only populates it for scalar
4806// reductions with a supported atomic operator.
4807static bool isAtomicableReductionSet(
4808 ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos) {
4809 return all_of(Range&: ReductionInfos, P: [](const OpenMPIRBuilder::ReductionInfo &RI) {
4810 return static_cast<bool>(RI.AtomicReductionGen);
4811 });
4812}
4813
4814OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createReductionsGPU(
4815 const LocationDescription &Loc, InsertPointTy AllocaIP,
4816 InsertPointTy CodeGenIP, ArrayRef<ReductionInfo> ReductionInfos,
4817 ArrayRef<bool> IsByRef, bool IsNoWait, bool IsTeamsReduction, bool IsSPMD,
4818 ReductionGenCBKind ReductionGenCBKind, std::optional<omp::GV> GridValue,
4819 Value *SrcLocInfo) {
4820 if (!updateToLocation(Loc))
4821 return InsertPointTy();
4822 Builder.restoreIP(IP: CodeGenIP);
4823 checkReductionInfos(ReductionInfos, /*IsGPU*/ true);
4824 LLVMContext &Ctx = M.getContext();
4825
4826 // Source location for the ident struct
4827 if (!SrcLocInfo) {
4828 uint32_t SrcLocStrSize;
4829 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
4830 SrcLocInfo = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
4831 }
4832
4833 if (ReductionInfos.size() == 0)
4834 return Builder.saveIP();
4835
4836 BasicBlock *ContinuationBlock = nullptr;
4837 if (ReductionGenCBKind != ReductionGenCBKind::Clang) {
4838 // Copied code from createReductions
4839 BasicBlock *InsertBlock = Loc.IP.getBlock();
4840 ContinuationBlock =
4841 InsertBlock->splitBasicBlock(I: Loc.IP.getPoint(), BBName: "reduce.finalize");
4842 InsertBlock->getTerminator()->eraseFromParent();
4843 Builder.SetInsertPoint(TheBB: InsertBlock, IP: InsertBlock->end());
4844 }
4845
4846 Function *CurFunc = Builder.GetInsertBlock()->getParent();
4847 AttributeList FuncAttrs;
4848 AttrBuilder AttrBldr(Ctx);
4849 for (auto Attr : CurFunc->getAttributes().getFnAttrs())
4850 AttrBldr.addAttribute(A: Attr);
4851 AttrBldr.removeAttribute(Val: Attribute::OptimizeNone);
4852 FuncAttrs = FuncAttrs.addFnAttributes(C&: Ctx, B: AttrBldr);
4853
4854 CodeGenIP = Builder.saveIP();
4855 Expected<Function *> ReductionResult = createReductionFunction(
4856 ReducerName: Builder.GetInsertBlock()->getParent()->getName(), ReductionInfos, IsByRef,
4857 ReductionGenCBKind, FuncAttrs);
4858 if (!ReductionResult)
4859 return ReductionResult.takeError();
4860 Function *ReductionFunc = *ReductionResult;
4861 Builder.restoreIP(IP: CodeGenIP);
4862
4863 // Set the grid value in the config needed for lowering later on
4864 if (GridValue.has_value())
4865 Config.setGridValue(GridValue.value());
4866 else
4867 Config.setGridValue(getGridValue(T, Kernel: ReductionFunc));
4868
4869 // Build res = __kmpc_reduce{_nowait}(<gtid>, <n>, sizeof(RedList),
4870 // RedList, shuffle_reduce_func, interwarp_copy_func);
4871 // or
4872 // Build res = __kmpc_reduce_teams_nowait_simple(<loc>, <gtid>, <lck>);
4873 Value *Res;
4874
4875 // 1. Build a list of reduction variables.
4876 // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
4877 auto Size = ReductionInfos.size();
4878 Type *PtrTy = PointerType::get(C&: Ctx, AddressSpace: Config.getDefaultTargetAS());
4879 Type *FuncPtrTy =
4880 Builder.getPtrTy(AddrSpace: M.getDataLayout().getProgramAddressSpace());
4881 Type *RedArrayTy = ArrayType::get(ElementType: PtrTy, NumElements: Size);
4882 CodeGenIP = Builder.saveIP();
4883 Builder.restoreIP(IP: AllocaIP);
4884 Value *ReductionListAlloca =
4885 Builder.CreateAlloca(Ty: RedArrayTy, ArraySize: nullptr, Name: ".omp.reduction.red_list");
4886 Value *ReductionList = Builder.CreatePointerBitCastOrAddrSpaceCast(
4887 V: ReductionListAlloca, DestTy: PtrTy, Name: ReductionListAlloca->getName() + ".ascast");
4888 Builder.restoreIP(IP: CodeGenIP);
4889 Type *IndexTy = Builder.getIndexTy(
4890 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
4891 for (auto En : enumerate(First&: ReductionInfos)) {
4892 const ReductionInfo &RI = En.value();
4893 Value *ElemPtr = Builder.CreateInBoundsGEP(
4894 Ty: RedArrayTy, Ptr: ReductionList,
4895 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
4896
4897 Value *PrivateVar = RI.PrivateVariable;
4898 bool IsByRefElem = !IsByRef.empty() && IsByRef[En.index()];
4899 if (IsByRefElem)
4900 PrivateVar = Builder.CreateLoad(Ty: RI.ElementType, Ptr: PrivateVar);
4901
4902 Value *CastElem =
4903 Builder.CreatePointerBitCastOrAddrSpaceCast(V: PrivateVar, DestTy: PtrTy);
4904 Builder.CreateStore(Val: CastElem, Ptr: ElemPtr);
4905 }
4906 CodeGenIP = Builder.saveIP();
4907 Expected<Function *> SarFunc = emitShuffleAndReduceFunction(
4908 ReductionInfos, ReduceFn: ReductionFunc, FuncAttrs, IsByRef);
4909
4910 if (!SarFunc)
4911 return SarFunc.takeError();
4912
4913 Expected<Function *> CopyResult =
4914 emitInterWarpCopyFunction(Loc, ReductionInfos, FuncAttrs, IsByRef);
4915 if (!CopyResult)
4916 return CopyResult.takeError();
4917 Function *WcFunc = *CopyResult;
4918 Builder.restoreIP(IP: CodeGenIP);
4919
4920 Value *RL = Builder.CreatePointerBitCastOrAddrSpaceCast(V: ReductionList, DestTy: PtrTy);
4921
4922 // NOTE: ReductionDataSize is passed as the reduce_data_size argument to
4923 // __kmpc_nvptx_parallel_reduce_nowait_v2, but the runtime implementations do
4924 // not currently use it. It is computed here conservatively as max(element
4925 // sizes) * N rather than the exact sum, which over-calculates the size for
4926 // mixed reduction types but is harmless given the argument is unused.
4927 // TODO: Consider dropping this computation if the runtime API is ever revised
4928 // to remove the unused parameter.
4929 unsigned MaxDataSize = 0;
4930 SmallVector<Type *> ReductionTypeArgs;
4931 for (auto En : enumerate(First&: ReductionInfos)) {
4932 // Use ByRefElementType for by-ref reductions so that MaxDataSize matches
4933 // the actual data size stored in the global reduction buffer, consistent
4934 // with the ReductionsBufferTy struct used for GEP offsets below.
4935 Type *RedTypeArg = (!IsByRef.empty() && IsByRef[En.index()])
4936 ? En.value().ByRefElementType
4937 : En.value().ElementType;
4938 auto Size = M.getDataLayout().getTypeStoreSize(Ty: RedTypeArg);
4939 if (Size > MaxDataSize)
4940 MaxDataSize = Size;
4941 ReductionTypeArgs.emplace_back(Args&: RedTypeArg);
4942 }
4943 Value *ReductionDataSize =
4944 Builder.getInt64(C: MaxDataSize * ReductionInfos.size());
4945
4946 // Helper function to copy thread-local data back to the original reduction
4947 // list.
4948 Function *CopyScratchToListFunc = nullptr;
4949 // Thread-local storage for the reduction variables.
4950 Value *ScratchForCopyBack = nullptr;
4951 // RL pointer to which the final value from the per-thread scratch should be
4952 // copied back. (Basically RL, appropriately casted if necessary.)
4953 Value *RLForCopyBack = RL;
4954
4955 bool IsAtomicReduction =
4956 IsTeamsReduction && isAtomicableReductionSet(ReductionInfos);
4957
4958 if (!IsTeamsReduction) {
4959 Value *SarFuncCast =
4960 Builder.CreatePointerBitCastOrAddrSpaceCast(V: *SarFunc, DestTy: FuncPtrTy);
4961 Value *WcFuncCast =
4962 Builder.CreatePointerBitCastOrAddrSpaceCast(V: WcFunc, DestTy: FuncPtrTy);
4963 Value *Args[] = {SrcLocInfo, ReductionDataSize, RL, SarFuncCast,
4964 WcFuncCast};
4965 Function *Pv2Ptr = getOrCreateRuntimeFunctionPtr(
4966 FnID: RuntimeFunction::OMPRTL___kmpc_nvptx_parallel_reduce_nowait_v2);
4967 Res = createRuntimeFunctionCall(Callee: Pv2Ptr, Args);
4968 } else if (IsAtomicReduction) {
4969 // Atomic cross-team reduction fast path: determine the team's main thread
4970 // that is later to fold its value atomically into the mapped variable.
4971 Function *IsMainThreadFn = getOrCreateRuntimeFunctionPtr(
4972 FnID: RuntimeFunction::OMPRTL___kmpc_is_team_main_thread);
4973 Res = createRuntimeFunctionCall(Callee: IsMainThreadFn, Args: {});
4974 } else {
4975 CodeGenIP = Builder.saveIP();
4976 StructType *ReductionsBufferTy = StructType::create(
4977 Context&: Ctx, Elements: ReductionTypeArgs, Name: "struct._globalized_locals_ty");
4978
4979 Expected<Function *> LtGCFunc = emitListToGlobalCopyFunction(
4980 ReductionInfos, ReductionsBufferTy, FuncAttrs, IsByRef);
4981 if (!LtGCFunc)
4982 return LtGCFunc.takeError();
4983
4984 Expected<Function *> GtLCFunc = emitGlobalToListCopyFunction(
4985 ReductionInfos, ReductionsBufferTy, FuncAttrs, IsByRef);
4986 if (!GtLCFunc)
4987 return GtLCFunc.takeError();
4988
4989 Expected<Function *> GtLRFunc = emitGlobalToListReduceFunction(
4990 ReductionInfos, ReduceFn: ReductionFunc, ReductionsBufferTy, FuncAttrs, IsByRef);
4991 if (!GtLRFunc)
4992 return GtLRFunc.takeError();
4993
4994 Builder.restoreIP(IP: CodeGenIP);
4995
4996 // The runtime's cross-team final aggregate uses the storage pointed at by
4997 // its reduce-list argument as per-thread scratch. When the surrounding
4998 // kernel is already in SPMD execution mode, clang emitted each reduction
4999 // private as a per-thread `alloca addrspace(5)`, so the original red_list
5000 // (RL) is already per-thread and nothing else is needed.
5001 //
5002 // When the kernel is in Non-SPMD execution mode at codegen time, clang's
5003 // Generic-mode globalization put the reduction private into team-shared
5004 // LDS. OpenMPOpt may later upgrade the kernel to Generic-SPMD, at which
5005 // point all threads of the last team would race on the shared LDS slot.
5006 // Emit a per-thread scratch buffer and a per-thread RL, copy the team-local
5007 // value in, and hand the per-thread RL to the runtime instead. The writer
5008 // thread copies the final value from that per-thread scratch back to RL
5009 // before running the existing combine path below.
5010
5011 // Thread-local RL (might need localization below before being passed to the
5012 // runtime).
5013 Value *RuntimeRL = RL;
5014
5015 if (!IsSPMD) {
5016 CodeGenIP = Builder.saveIP();
5017 Builder.restoreIP(IP: AllocaIP);
5018 // Allocate thread-local buffer for the reduction variables.
5019 Value *PerThreadScratchAlloca = Builder.CreateAlloca(
5020 Ty: ReductionsBufferTy, /*ArraySize=*/nullptr, Name: ".omp.reduction.scratch");
5021 Value *PerThreadScratch = Builder.CreatePointerBitCastOrAddrSpaceCast(
5022 V: PerThreadScratchAlloca, DestTy: PtrTy,
5023 Name: PerThreadScratchAlloca->getName() + ".ascast");
5024 // Allocate thread-local buffer for the pointers to the reduction
5025 // variables.
5026 Value *PerThreadRedListAlloca =
5027 Builder.CreateAlloca(Ty: RedArrayTy, /*ArraySize=*/nullptr,
5028 Name: ".omp.reduction.per_thread_red_list");
5029 RuntimeRL = Builder.CreatePointerBitCastOrAddrSpaceCast(
5030 V: PerThreadRedListAlloca, DestTy: PtrTy,
5031 Name: PerThreadRedListAlloca->getName() + ".ascast");
5032 Builder.restoreIP(IP: CodeGenIP);
5033
5034 // Iterate over the reduction variables and copy the team-local value to
5035 // the thread-local buffer.
5036 for (auto En : enumerate(First&: ReductionInfos)) {
5037 const ReductionInfo &RI = En.value();
5038 bool IsByRefElem = !IsByRef.empty() && IsByRef[En.index()];
5039
5040 Value *FieldPtr = Builder.CreateConstInBoundsGEP2_32(
5041 Ty: ReductionsBufferTy, Ptr: PerThreadScratch, Idx0: 0, Idx1: En.index());
5042 Value *Slot = Builder.CreateConstInBoundsGEP2_32(Ty: RedArrayTy, Ptr: RuntimeRL,
5043 Idx0: 0, Idx1: En.index());
5044
5045 Value *RuntimeListEntry = FieldPtr;
5046 if (IsByRefElem && RI.DataPtrPtrGen) {
5047 Value *SrcDescriptor =
5048 Builder.CreateLoad(Ty: RI.ElementType, Ptr: RI.PrivateVariable);
5049 Expected<Value *> Descriptor = createReductionDescriptorCopy(
5050 AllocaIP, RI, DataPtr: FieldPtr, SrcDescriptorAddr: SrcDescriptor, DescriptorPtrTy: PtrTy);
5051 if (!Descriptor)
5052 return Descriptor.takeError();
5053 RuntimeListEntry = *Descriptor;
5054 }
5055 Builder.CreateStore(Val: RuntimeListEntry, Ptr: Slot);
5056 }
5057 // The copy helpers were emitted with default-AS (AS 0) pointer params
5058 // (see emitListToGlobalCopyFunction / emitGlobalToListCopyFunction),
5059 // but PerThreadScratch and RL live in the target's default AS, which
5060 // is non-zero on e.g. SPIRV. (See Config.getDefaultTargetAS().)
5061 Type *CopyArg0Ty = (*LtGCFunc)->getFunctionType()->getParamType(i: 0);
5062 Type *CopyArg2Ty = (*LtGCFunc)->getFunctionType()->getParamType(i: 2);
5063 ScratchForCopyBack = Builder.CreatePointerBitCastOrAddrSpaceCast(
5064 V: PerThreadScratch, DestTy: CopyArg0Ty);
5065 RLForCopyBack =
5066 Builder.CreatePointerBitCastOrAddrSpaceCast(V: RL, DestTy: CopyArg2Ty);
5067 // Use index 0 because there is no array of target values to index into,
5068 // there is only one thread-local memory slot.
5069 // restoreIP above left a stale/empty debug location; this inlinable call
5070 // to a debug-info-bearing helper needs one or the verifier rejects the
5071 // module ("!dbg attachment points at wrong subprogram") after inlining.
5072 Builder.SetCurrentDebugLocation(Loc.DL);
5073 Builder.CreateCall(
5074 Callee: *LtGCFunc, Args: {ScratchForCopyBack, Builder.getInt32(C: 0), RLForCopyBack});
5075 CopyScratchToListFunc = *GtLCFunc;
5076 }
5077
5078 Value *Args3[] = {SrcLocInfo, RuntimeRL, *SarFunc, WcFunc,
5079 *LtGCFunc, *GtLCFunc, *GtLRFunc};
5080
5081 Function *TeamsReduceFn = getOrCreateRuntimeFunctionPtr(
5082 FnID: RuntimeFunction::OMPRTL___kmpc_gpu_xteam_reduce_nowait);
5083 Res = createRuntimeFunctionCall(Callee: TeamsReduceFn, Args: Args3);
5084 }
5085
5086 // 5. Build if (res == 1)
5087 BasicBlock *ExitBB = BasicBlock::Create(Context&: Ctx, Name: ".omp.reduction.done");
5088 BasicBlock *ThenBB = BasicBlock::Create(Context&: Ctx, Name: ".omp.reduction.then");
5089 Value *Cond = Builder.CreateICmpEQ(LHS: Res, RHS: Builder.getInt32(C: 1));
5090 Builder.CreateCondBr(Cond, True: ThenBB, False: ExitBB);
5091
5092 // 6. Build then branch: where we have reduced values in the master
5093 // thread in each team.
5094 // __kmpc_end_reduce{_nowait}(<gtid>);
5095 // break;
5096 emitBlock(BB: ThenBB, CurFn: CurFunc);
5097
5098 // Copy the writer thread's per-thread scratch result back into the original
5099 // red-list storage before the existing combine path reads RI.PrivateVariable.
5100 // Set a debug location: this inlinable call to a debug-info-bearing helper
5101 // needs one or the verifier rejects the module after inlining.
5102 if (ScratchForCopyBack) {
5103 Builder.SetCurrentDebugLocation(Loc.DL);
5104 Builder.CreateCall(
5105 Callee: CopyScratchToListFunc,
5106 Args: {ScratchForCopyBack, Builder.getInt32(C: 0), RLForCopyBack});
5107 }
5108
5109 // Add emission of __kmpc_end_reduce{_nowait}(<gtid>);
5110 for (auto En : enumerate(First&: ReductionInfos)) {
5111 const ReductionInfo &RI = En.value();
5112
5113 // Atomic cross-team fast path: each team's main thread folds its
5114 // team-reduced value directly into the mapped reduction variable with a
5115 // single atomicrmw.
5116 if (IsAtomicReduction) {
5117 InsertPointOrErrorTy AfterIP = RI.AtomicReductionGen(
5118 Builder.saveIP(), RI.ElementType, RI.Variable, RI.PrivateVariable);
5119 if (!AfterIP)
5120 return AfterIP.takeError();
5121 Builder.restoreIP(IP: *AfterIP);
5122 continue;
5123 }
5124
5125 Type *ValueType = RI.ElementType;
5126 Value *RedValue = RI.Variable;
5127
5128 Value *RHS =
5129 Builder.CreatePointerBitCastOrAddrSpaceCast(V: RI.PrivateVariable, DestTy: PtrTy);
5130
5131 if (ReductionGenCBKind == ReductionGenCBKind::Clang) {
5132 Value *LHSPtr, *RHSPtr;
5133 Builder.restoreIP(IP: RI.ReductionGenClang(Builder.saveIP(), En.index(),
5134 &LHSPtr, &RHSPtr, CurFunc));
5135
5136 // Fix the CallBack code genereated to use the correct Values for the LHS
5137 // and RHS. Cast to match types before replacing (necessary to handle
5138 // different address spaces).
5139 if (LHSPtr->getType() != RedValue->getType())
5140 RedValue = Builder.CreatePointerBitCastOrAddrSpaceCast(
5141 V: RedValue, DestTy: LHSPtr->getType());
5142 if (RHSPtr->getType() != RHS->getType())
5143 RHS =
5144 Builder.CreatePointerBitCastOrAddrSpaceCast(V: RHS, DestTy: RHSPtr->getType());
5145
5146 LHSPtr->replaceUsesWithIf(New: RedValue, ShouldReplace: [ReductionFunc](const Use &U) {
5147 return cast<Instruction>(Val: U.getUser())->getParent()->getParent() ==
5148 ReductionFunc;
5149 });
5150 RHSPtr->replaceUsesWithIf(New: RHS, ShouldReplace: [ReductionFunc](const Use &U) {
5151 return cast<Instruction>(Val: U.getUser())->getParent()->getParent() ==
5152 ReductionFunc;
5153 });
5154 } else {
5155 if (IsByRef.empty() || !IsByRef[En.index()]) {
5156 RedValue = Builder.CreateLoad(Ty: ValueType, Ptr: RI.Variable,
5157 Name: "red.value." + Twine(En.index()));
5158 }
5159 Value *PrivateRedValue = Builder.CreateLoad(
5160 Ty: ValueType, Ptr: RHS, Name: "red.private.value" + Twine(En.index()));
5161 Value *Reduced;
5162 InsertPointOrErrorTy AfterIP =
5163 RI.ReductionGen(Builder.saveIP(), RedValue, PrivateRedValue, Reduced);
5164 if (!AfterIP)
5165 return AfterIP.takeError();
5166 Builder.restoreIP(IP: *AfterIP);
5167
5168 if (!IsByRef.empty() && !IsByRef[En.index()])
5169 Builder.CreateStore(Val: Reduced, Ptr: RI.Variable);
5170 }
5171 }
5172 emitBlock(BB: ExitBB, CurFn: CurFunc);
5173 if (ContinuationBlock) {
5174 Builder.CreateBr(Dest: ContinuationBlock);
5175 Builder.SetInsertPoint(ContinuationBlock);
5176 }
5177 Config.setEmitLLVMUsed();
5178
5179 return Builder.saveIP();
5180}
5181
5182static Function *getFreshReductionFunc(Module &M) {
5183 Type *VoidTy = Type::getVoidTy(C&: M.getContext());
5184 Type *Int8PtrTy = PointerType::getUnqual(C&: M.getContext());
5185 auto *FuncTy =
5186 FunctionType::get(Result: VoidTy, Params: {Int8PtrTy, Int8PtrTy}, /* IsVarArg */ isVarArg: false);
5187 return Function::Create(Ty: FuncTy, Linkage: GlobalVariable::InternalLinkage,
5188 N: ".omp.reduction.func", M: &M);
5189}
5190
5191static Error populateReductionFunction(
5192 Function *ReductionFunc,
5193 ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos,
5194 IRBuilder<> &Builder, ArrayRef<bool> IsByRef, bool IsGPU) {
5195 IRBuilder<>::InsertPointGuard IPG(Builder);
5196 Module *Module = ReductionFunc->getParent();
5197 BasicBlock *ReductionFuncBlock =
5198 BasicBlock::Create(Context&: Module->getContext(), Name: "", Parent: ReductionFunc);
5199 Builder.SetInsertPoint(ReductionFuncBlock);
5200 Builder.SetCurrentDebugLocation(llvm::DebugLoc());
5201 Value *LHSArrayPtr = nullptr;
5202 Value *RHSArrayPtr = nullptr;
5203 if (IsGPU) {
5204 // Need to alloca memory here and deal with the pointers before getting
5205 // LHS/RHS pointers out
5206 //
5207 Argument *Arg0 = ReductionFunc->getArg(i: 0);
5208 Argument *Arg1 = ReductionFunc->getArg(i: 1);
5209 Type *Arg0Type = Arg0->getType();
5210 Type *Arg1Type = Arg1->getType();
5211
5212 Value *LHSAlloca =
5213 Builder.CreateAlloca(Ty: Arg0Type, ArraySize: nullptr, Name: Arg0->getName() + ".addr");
5214 Value *RHSAlloca =
5215 Builder.CreateAlloca(Ty: Arg1Type, ArraySize: nullptr, Name: Arg1->getName() + ".addr");
5216 Value *LHSAddrCast =
5217 Builder.CreatePointerBitCastOrAddrSpaceCast(V: LHSAlloca, DestTy: Arg0Type);
5218 Value *RHSAddrCast =
5219 Builder.CreatePointerBitCastOrAddrSpaceCast(V: RHSAlloca, DestTy: Arg1Type);
5220 Builder.CreateStore(Val: Arg0, Ptr: LHSAddrCast);
5221 Builder.CreateStore(Val: Arg1, Ptr: RHSAddrCast);
5222 LHSArrayPtr = Builder.CreateLoad(Ty: Arg0Type, Ptr: LHSAddrCast);
5223 RHSArrayPtr = Builder.CreateLoad(Ty: Arg1Type, Ptr: RHSAddrCast);
5224 } else {
5225 LHSArrayPtr = ReductionFunc->getArg(i: 0);
5226 RHSArrayPtr = ReductionFunc->getArg(i: 1);
5227 }
5228
5229 unsigned NumReductions = ReductionInfos.size();
5230 Type *RedArrayTy = ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: NumReductions);
5231
5232 for (auto En : enumerate(First&: ReductionInfos)) {
5233 const OpenMPIRBuilder::ReductionInfo &RI = En.value();
5234 Value *LHSI8PtrPtr = Builder.CreateConstInBoundsGEP2_64(
5235 Ty: RedArrayTy, Ptr: LHSArrayPtr, Idx0: 0, Idx1: En.index());
5236 Value *LHSI8Ptr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: LHSI8PtrPtr);
5237 Value *LHSPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
5238 V: LHSI8Ptr, DestTy: RI.Variable->getType());
5239 Value *LHS = Builder.CreateLoad(Ty: RI.ElementType, Ptr: LHSPtr);
5240 Value *RHSI8PtrPtr = Builder.CreateConstInBoundsGEP2_64(
5241 Ty: RedArrayTy, Ptr: RHSArrayPtr, Idx0: 0, Idx1: En.index());
5242 Value *RHSI8Ptr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: RHSI8PtrPtr);
5243 Value *RHSPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
5244 V: RHSI8Ptr, DestTy: RI.PrivateVariable->getType());
5245 Value *RHS = Builder.CreateLoad(Ty: RI.ElementType, Ptr: RHSPtr);
5246 Value *Reduced;
5247 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
5248 RI.ReductionGen(Builder.saveIP(), LHS, RHS, Reduced);
5249 if (!AfterIP)
5250 return AfterIP.takeError();
5251
5252 Builder.restoreIP(IP: *AfterIP);
5253 // TODO: Consider flagging an error.
5254 if (!Builder.GetInsertBlock())
5255 return Error::success();
5256
5257 // store is inside of the reduction region when using by-ref
5258 if (!IsByRef[En.index()])
5259 Builder.CreateStore(Val: Reduced, Ptr: LHSPtr);
5260 }
5261 Builder.CreateRetVoid();
5262 return Error::success();
5263}
5264
5265OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createReductions(
5266 const LocationDescription &Loc, InsertPointTy AllocaIP,
5267 ArrayRef<ReductionInfo> ReductionInfos, ArrayRef<bool> IsByRef,
5268 bool IsNoWait, bool IsTeamsReduction) {
5269 assert(ReductionInfos.size() == IsByRef.size());
5270 if (Config.isGPU())
5271 return createReductionsGPU(Loc, AllocaIP, CodeGenIP: Builder.saveIP(), ReductionInfos,
5272 IsByRef, IsNoWait, IsTeamsReduction);
5273
5274 checkReductionInfos(ReductionInfos, /*IsGPU*/ false);
5275
5276 if (!updateToLocation(Loc))
5277 return InsertPointTy();
5278
5279 if (ReductionInfos.size() == 0)
5280 return Builder.saveIP();
5281
5282 BasicBlock *InsertBlock = Loc.IP.getBlock();
5283 BasicBlock *ContinuationBlock =
5284 InsertBlock->splitBasicBlock(I: Loc.IP.getPoint(), BBName: "reduce.finalize");
5285 InsertBlock->getTerminator()->eraseFromParent();
5286
5287 // Create and populate array of type-erased pointers to private reduction
5288 // values.
5289 unsigned NumReductions = ReductionInfos.size();
5290 Type *RedArrayTy = ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: NumReductions);
5291 Builder.SetInsertPoint(AllocaIP.getBlock()->getTerminator());
5292 Value *RedArray = Builder.CreateAlloca(Ty: RedArrayTy, ArraySize: nullptr, Name: "red.array");
5293
5294 Builder.SetInsertPoint(TheBB: InsertBlock, IP: InsertBlock->end());
5295 // Emitting the alloca moved the insertion point into the alloca block and
5296 // can clear the debug loc. Restore back to Loc.DL.
5297 Builder.SetCurrentDebugLocation(Loc.DL);
5298
5299 for (auto En : enumerate(First&: ReductionInfos)) {
5300 unsigned Index = En.index();
5301 const ReductionInfo &RI = En.value();
5302 Value *RedArrayElemPtr = Builder.CreateConstInBoundsGEP2_64(
5303 Ty: RedArrayTy, Ptr: RedArray, Idx0: 0, Idx1: Index, Name: "red.array.elem." + Twine(Index));
5304 Builder.CreateStore(Val: RI.PrivateVariable, Ptr: RedArrayElemPtr);
5305 }
5306
5307 // Emit a call to the runtime function that orchestrates the reduction.
5308 // Declare the reduction function in the process.
5309 Type *IndexTy = Builder.getIndexTy(
5310 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
5311 Function *Func = Builder.GetInsertBlock()->getParent();
5312 Module *Module = Func->getParent();
5313 uint32_t SrcLocStrSize;
5314 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
5315 bool CanGenerateAtomic = all_of(Range&: ReductionInfos, P: [](const ReductionInfo &RI) {
5316 return RI.AtomicReductionGen;
5317 });
5318 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize,
5319 LocFlags: CanGenerateAtomic
5320 ? IdentFlag::OMP_IDENT_FLAG_ATOMIC_REDUCE
5321 : IdentFlag(0));
5322 Value *ThreadId = getOrCreateThreadID(Ident);
5323 Constant *NumVariables = Builder.getInt32(C: NumReductions);
5324 const DataLayout &DL = Module->getDataLayout();
5325 unsigned RedArrayByteSize = DL.getTypeStoreSize(Ty: RedArrayTy);
5326 Constant *RedArraySize = ConstantInt::get(Ty: IndexTy, V: RedArrayByteSize);
5327 Function *ReductionFunc = getFreshReductionFunc(M&: *Module);
5328 Value *Lock = getOMPCriticalRegionLock(CriticalName: ".reduction");
5329 Function *ReduceFunc = getOrCreateRuntimeFunctionPtr(
5330 FnID: IsNoWait ? RuntimeFunction::OMPRTL___kmpc_reduce_nowait
5331 : RuntimeFunction::OMPRTL___kmpc_reduce);
5332 CallInst *ReduceCall =
5333 createRuntimeFunctionCall(Callee: ReduceFunc,
5334 Args: {Ident, ThreadId, NumVariables, RedArraySize,
5335 RedArray, ReductionFunc, Lock},
5336 Name: "reduce");
5337
5338 // Create final reduction entry blocks for the atomic and non-atomic case.
5339 // Emit IR that dispatches control flow to one of the blocks based on the
5340 // reduction supporting the atomic mode.
5341 BasicBlock *NonAtomicRedBlock =
5342 BasicBlock::Create(Context&: Module->getContext(), Name: "reduce.switch.nonatomic", Parent: Func);
5343 BasicBlock *AtomicRedBlock =
5344 BasicBlock::Create(Context&: Module->getContext(), Name: "reduce.switch.atomic", Parent: Func);
5345 SwitchInst *Switch =
5346 Builder.CreateSwitch(V: ReduceCall, Dest: ContinuationBlock, /* NumCases */ 2);
5347 Switch->addCase(OnVal: Builder.getInt32(C: 1), Dest: NonAtomicRedBlock);
5348 Switch->addCase(OnVal: Builder.getInt32(C: 2), Dest: AtomicRedBlock);
5349
5350 // Populate the non-atomic reduction using the elementwise reduction function.
5351 // This loads the elements from the global and private variables and reduces
5352 // them before storing back the result to the global variable.
5353 Builder.SetInsertPoint(NonAtomicRedBlock);
5354 for (auto En : enumerate(First&: ReductionInfos)) {
5355 const ReductionInfo &RI = En.value();
5356 Type *ValueType = RI.ElementType;
5357 // We have one less load for by-ref case because that load is now inside of
5358 // the reduction region
5359 Value *RedValue = RI.Variable;
5360 if (!IsByRef[En.index()]) {
5361 RedValue = Builder.CreateLoad(Ty: ValueType, Ptr: RI.Variable,
5362 Name: "red.value." + Twine(En.index()));
5363 }
5364 Value *PrivateRedValue =
5365 Builder.CreateLoad(Ty: ValueType, Ptr: RI.PrivateVariable,
5366 Name: "red.private.value." + Twine(En.index()));
5367 Value *Reduced;
5368 InsertPointOrErrorTy AfterIP =
5369 RI.ReductionGen(Builder.saveIP(), RedValue, PrivateRedValue, Reduced);
5370 if (!AfterIP)
5371 return AfterIP.takeError();
5372 Builder.restoreIP(IP: *AfterIP);
5373
5374 if (!Builder.GetInsertBlock())
5375 return InsertPointTy();
5376 // for by-ref case, the load is inside of the reduction region
5377 if (!IsByRef[En.index()])
5378 Builder.CreateStore(Val: Reduced, Ptr: RI.Variable);
5379 }
5380 Function *EndReduceFunc = getOrCreateRuntimeFunctionPtr(
5381 FnID: IsNoWait ? RuntimeFunction::OMPRTL___kmpc_end_reduce_nowait
5382 : RuntimeFunction::OMPRTL___kmpc_end_reduce);
5383 createRuntimeFunctionCall(Callee: EndReduceFunc, Args: {Ident, ThreadId, Lock});
5384 Builder.CreateBr(Dest: ContinuationBlock);
5385
5386 // Populate the atomic reduction using the atomic elementwise reduction
5387 // function. There are no loads/stores here because they will be happening
5388 // inside the atomic elementwise reduction.
5389 Builder.SetInsertPoint(AtomicRedBlock);
5390 if (CanGenerateAtomic && llvm::none_of(Range&: IsByRef, P: [](bool P) { return P; })) {
5391 for (const ReductionInfo &RI : ReductionInfos) {
5392 InsertPointOrErrorTy AfterIP = RI.AtomicReductionGen(
5393 Builder.saveIP(), RI.ElementType, RI.Variable, RI.PrivateVariable);
5394 if (!AfterIP)
5395 return AfterIP.takeError();
5396 Builder.restoreIP(IP: *AfterIP);
5397 if (!Builder.GetInsertBlock())
5398 return InsertPointTy();
5399 }
5400 Builder.CreateBr(Dest: ContinuationBlock);
5401 } else {
5402 Builder.CreateUnreachable();
5403 }
5404
5405 // Populate the outlined reduction function using the elementwise reduction
5406 // function. Partial values are extracted from the type-erased array of
5407 // pointers to private variables.
5408 Error Err = populateReductionFunction(ReductionFunc, ReductionInfos, Builder,
5409 IsByRef, /*isGPU=*/IsGPU: false);
5410 if (Err)
5411 return Err;
5412
5413 if (!Builder.GetInsertBlock())
5414 return InsertPointTy();
5415
5416 Builder.SetInsertPoint(ContinuationBlock);
5417 return Builder.saveIP();
5418}
5419
5420OpenMPIRBuilder::InsertPointOrErrorTy
5421OpenMPIRBuilder::createMaster(const LocationDescription &Loc,
5422 BodyGenCallbackTy BodyGenCB,
5423 FinalizeCallbackTy FiniCB) {
5424 if (!updateToLocation(Loc))
5425 return Loc.IP;
5426
5427 Directive OMPD = Directive::OMPD_master;
5428 uint32_t SrcLocStrSize;
5429 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
5430 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
5431 Value *ThreadId = getOrCreateThreadID(Ident);
5432 Value *Args[] = {Ident, ThreadId};
5433
5434 Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_master);
5435 Instruction *EntryCall = createRuntimeFunctionCall(Callee: EntryRTLFn, Args);
5436
5437 Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_end_master);
5438 Instruction *ExitCall = createRuntimeFunctionCall(Callee: ExitRTLFn, Args);
5439
5440 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
5441 /*Conditional*/ true, /*hasFinalize*/ HasFinalize: true);
5442}
5443
5444OpenMPIRBuilder::InsertPointOrErrorTy
5445OpenMPIRBuilder::createMasked(const LocationDescription &Loc,
5446 BodyGenCallbackTy BodyGenCB,
5447 FinalizeCallbackTy FiniCB, Value *Filter) {
5448 IRBuilder<>::InsertPointGuard IPG(Builder);
5449 if (!updateToLocation(Loc))
5450 return Loc.IP;
5451
5452 Directive OMPD = Directive::OMPD_masked;
5453 uint32_t SrcLocStrSize;
5454 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
5455 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
5456 Value *ThreadId = getOrCreateThreadID(Ident);
5457 Value *Args[] = {Ident, ThreadId, Filter};
5458 Value *ArgsEnd[] = {Ident, ThreadId};
5459
5460 Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_masked);
5461 Instruction *EntryCall = createRuntimeFunctionCall(Callee: EntryRTLFn, Args);
5462
5463 Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_end_masked);
5464 Instruction *ExitCall = createRuntimeFunctionCall(Callee: ExitRTLFn, Args: ArgsEnd);
5465
5466 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
5467 /*Conditional*/ true, /*hasFinalize*/ HasFinalize: true);
5468}
5469
5470static llvm::CallInst *emitNoUnwindRuntimeCall(IRBuilder<> &Builder,
5471 llvm::FunctionCallee Callee,
5472 ArrayRef<llvm::Value *> Args,
5473 const llvm::Twine &Name) {
5474 llvm::CallInst *Call = Builder.CreateCall(
5475 Callee, Args, OpBundles: SmallVector<llvm::OperandBundleDef, 1>(), Name);
5476 Call->setDoesNotThrow();
5477 return Call;
5478}
5479
5480// Expects input basic block is dominated by BeforeScanBB.
5481// Once Scan directive is encountered, the code after scan directive should be
5482// dominated by AfterScanBB. Scan directive splits the code sequence to
5483// scan and input phase. Based on whether inclusive or exclusive
5484// clause is used in the scan directive and whether input loop or scan loop
5485// is lowered, it adds jumps to input and scan phase. First Scan loop is the
5486// input loop and second is the scan loop. The code generated handles only
5487// inclusive scans now.
5488OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createScan(
5489 const LocationDescription &Loc, InsertPointTy AllocaIP,
5490 ArrayRef<llvm::Value *> ScanVars, ArrayRef<llvm::Type *> ScanVarsType,
5491 bool IsInclusive, ScanInfo *ScanRedInfo) {
5492 if (ScanRedInfo->OMPFirstScanLoop) {
5493 llvm::Error Err = emitScanBasedDirectiveDeclsIR(AllocaIP, ScanVars,
5494 ScanVarsType, ScanRedInfo);
5495 if (Err)
5496 return Err;
5497 }
5498 if (!updateToLocation(Loc))
5499 return Loc.IP;
5500
5501 llvm::Value *IV = ScanRedInfo->IV;
5502
5503 if (ScanRedInfo->OMPFirstScanLoop) {
5504 // Emit buffer[i] = red; at the end of the input phase.
5505 for (size_t i = 0; i < ScanVars.size(); i++) {
5506 Value *BuffPtr = (*(ScanRedInfo->ScanBuffPtrs))[ScanVars[i]];
5507 Value *Buff = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: BuffPtr);
5508 Type *DestTy = ScanVarsType[i];
5509 Value *Val = Builder.CreateInBoundsGEP(Ty: DestTy, Ptr: Buff, IdxList: IV, Name: "arrayOffset");
5510 Value *Src = Builder.CreateLoad(Ty: DestTy, Ptr: ScanVars[i]);
5511
5512 Builder.CreateStore(Val: Src, Ptr: Val);
5513 }
5514 }
5515 Builder.CreateBr(Dest: ScanRedInfo->OMPScanLoopExit);
5516 emitBlock(BB: ScanRedInfo->OMPScanDispatch,
5517 CurFn: Builder.GetInsertBlock()->getParent());
5518
5519 if (!ScanRedInfo->OMPFirstScanLoop) {
5520 IV = ScanRedInfo->IV;
5521 // Emit red = buffer[i]; at the entrance to the scan phase.
5522 // TODO: if exclusive scan, the red = buffer[i-1] needs to be updated.
5523 for (size_t i = 0; i < ScanVars.size(); i++) {
5524 Value *BuffPtr = (*(ScanRedInfo->ScanBuffPtrs))[ScanVars[i]];
5525 Value *Buff = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: BuffPtr);
5526 Type *DestTy = ScanVarsType[i];
5527 Value *SrcPtr =
5528 Builder.CreateInBoundsGEP(Ty: DestTy, Ptr: Buff, IdxList: IV, Name: "arrayOffset");
5529 Value *Src = Builder.CreateLoad(Ty: DestTy, Ptr: SrcPtr);
5530 Builder.CreateStore(Val: Src, Ptr: ScanVars[i]);
5531 }
5532 }
5533
5534 // TODO: Update it to CreateBr and remove dead blocks
5535 llvm::Value *CmpI = Builder.getInt1(V: true);
5536 if (ScanRedInfo->OMPFirstScanLoop == IsInclusive) {
5537 Builder.CreateCondBr(Cond: CmpI, True: ScanRedInfo->OMPBeforeScanBlock,
5538 False: ScanRedInfo->OMPAfterScanBlock);
5539 } else {
5540 Builder.CreateCondBr(Cond: CmpI, True: ScanRedInfo->OMPAfterScanBlock,
5541 False: ScanRedInfo->OMPBeforeScanBlock);
5542 }
5543 emitBlock(BB: ScanRedInfo->OMPAfterScanBlock,
5544 CurFn: Builder.GetInsertBlock()->getParent());
5545 Builder.SetInsertPoint(ScanRedInfo->OMPAfterScanBlock);
5546 return Builder.saveIP();
5547}
5548
5549Error OpenMPIRBuilder::emitScanBasedDirectiveDeclsIR(
5550 InsertPointTy AllocaIP, ArrayRef<Value *> ScanVars,
5551 ArrayRef<Type *> ScanVarsType, ScanInfo *ScanRedInfo) {
5552
5553 Builder.restoreIP(IP: AllocaIP);
5554 // Create the shared pointer at alloca IP.
5555 for (size_t i = 0; i < ScanVars.size(); i++) {
5556 llvm::Value *BuffPtr =
5557 Builder.CreateAlloca(Ty: Builder.getPtrTy(), ArraySize: nullptr, Name: "vla");
5558 (*(ScanRedInfo->ScanBuffPtrs))[ScanVars[i]] = BuffPtr;
5559 }
5560
5561 // Allocate temporary buffer by master thread
5562 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
5563 ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
5564 Builder.restoreIP(IP: CodeGenIP);
5565 Value *AllocSpan =
5566 Builder.CreateAdd(LHS: ScanRedInfo->Span, RHS: Builder.getInt32(C: 1));
5567 for (size_t i = 0; i < ScanVars.size(); i++) {
5568 Type *IntPtrTy = Builder.getInt32Ty();
5569 Value *Allocsize = Builder.CreateTypeSize(
5570 Ty: IntPtrTy, Size: M.getDataLayout().getTypeAllocSize(Ty: ScanVarsType[i]));
5571 Value *Buff =
5572 Builder.CreateMalloc(IntPtrTy, AllocSize: Allocsize, ArraySize: AllocSpan, MallocF: nullptr, Name: "arr");
5573 Builder.CreateStore(Val: Buff, Ptr: (*(ScanRedInfo->ScanBuffPtrs))[ScanVars[i]]);
5574 }
5575 return Error::success();
5576 };
5577 // TODO: Perform finalization actions for variables. This has to be
5578 // called for variables which have destructors/finalizers.
5579 auto FiniCB = [&](InsertPointTy CodeGenIP) { return llvm::Error::success(); };
5580
5581 Builder.SetInsertPoint(ScanRedInfo->OMPScanInit->getTerminator());
5582 llvm::Value *FilterVal = Builder.getInt32(C: 0);
5583 llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
5584 createMasked(Loc: Builder, BodyGenCB, FiniCB, Filter: FilterVal);
5585
5586 if (!AfterIP)
5587 return AfterIP.takeError();
5588 Builder.restoreIP(IP: *AfterIP);
5589 BasicBlock *InputBB = Builder.GetInsertBlock();
5590 if (InputBB->hasTerminator())
5591 Builder.SetInsertPoint(InputBB->getTerminator());
5592 AfterIP = createBarrier(Loc: Builder, Kind: llvm::omp::OMPD_barrier);
5593 if (!AfterIP)
5594 return AfterIP.takeError();
5595 Builder.restoreIP(IP: *AfterIP);
5596
5597 return Error::success();
5598}
5599
5600Error OpenMPIRBuilder::emitScanBasedDirectiveFinalsIR(
5601 ArrayRef<ReductionInfo> ReductionInfos, ScanInfo *ScanRedInfo) {
5602 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
5603 ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
5604 Builder.restoreIP(IP: CodeGenIP);
5605 for (ReductionInfo RedInfo : ReductionInfos) {
5606 Value *PrivateVar = RedInfo.PrivateVariable;
5607 Value *OrigVar = RedInfo.Variable;
5608 Value *BuffPtr = (*(ScanRedInfo->ScanBuffPtrs))[PrivateVar];
5609 Value *Buff = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: BuffPtr);
5610
5611 Type *SrcTy = RedInfo.ElementType;
5612 Value *Val = Builder.CreateInBoundsGEP(Ty: SrcTy, Ptr: Buff, IdxList: ScanRedInfo->Span,
5613 Name: "arrayOffset");
5614 Value *Src = Builder.CreateLoad(Ty: SrcTy, Ptr: Val);
5615
5616 Builder.CreateStore(Val: Src, Ptr: OrigVar);
5617 Builder.CreateFree(Source: Buff);
5618 }
5619 return Error::success();
5620 };
5621 // TODO: Perform finalization actions for variables. This has to be
5622 // called for variables which have destructors/finalizers.
5623 auto FiniCB = [&](InsertPointTy CodeGenIP) { return llvm::Error::success(); };
5624
5625 if (Instruction *TI = ScanRedInfo->OMPScanFinish->getTerminatorOrNull())
5626 Builder.SetInsertPoint(TI);
5627 else
5628 Builder.SetInsertPoint(ScanRedInfo->OMPScanFinish);
5629
5630 llvm::Value *FilterVal = Builder.getInt32(C: 0);
5631 llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
5632 createMasked(Loc: Builder, BodyGenCB, FiniCB, Filter: FilterVal);
5633
5634 if (!AfterIP)
5635 return AfterIP.takeError();
5636 Builder.restoreIP(IP: *AfterIP);
5637 BasicBlock *InputBB = Builder.GetInsertBlock();
5638 if (InputBB->hasTerminator())
5639 Builder.SetInsertPoint(InputBB->getTerminator());
5640 AfterIP = createBarrier(Loc: Builder, Kind: llvm::omp::OMPD_barrier);
5641 if (!AfterIP)
5642 return AfterIP.takeError();
5643 Builder.restoreIP(IP: *AfterIP);
5644 return Error::success();
5645}
5646
5647OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
5648 const LocationDescription &Loc,
5649 ArrayRef<llvm::OpenMPIRBuilder::ReductionInfo> ReductionInfos,
5650 ScanInfo *ScanRedInfo) {
5651
5652 if (!updateToLocation(Loc))
5653 return Loc.IP;
5654 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
5655 ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
5656 Builder.restoreIP(IP: CodeGenIP);
5657 Function *CurFn = Builder.GetInsertBlock()->getParent();
5658 // for (int k = 0; k <= ceil(log2(n)); ++k)
5659 llvm::BasicBlock *LoopBB =
5660 BasicBlock::Create(Context&: CurFn->getContext(), Name: "omp.outer.log.scan.body");
5661 llvm::BasicBlock *ExitBB =
5662 splitBB(Builder, CreateBranch: false, Name: "omp.outer.log.scan.exit");
5663 llvm::Function *F = llvm::Intrinsic::getOrInsertDeclaration(
5664 M: Builder.GetInsertBlock()->getModule(),
5665 id: (llvm::Intrinsic::ID)llvm::Intrinsic::log2, OverloadTys: Builder.getDoubleTy());
5666 llvm::BasicBlock *InputBB = Builder.GetInsertBlock();
5667 llvm::Value *Arg =
5668 Builder.CreateUIToFP(V: ScanRedInfo->Span, DestTy: Builder.getDoubleTy());
5669 llvm::Value *LogVal = emitNoUnwindRuntimeCall(Builder, Callee: F, Args: Arg, Name: "");
5670 F = llvm::Intrinsic::getOrInsertDeclaration(
5671 M: Builder.GetInsertBlock()->getModule(),
5672 id: (llvm::Intrinsic::ID)llvm::Intrinsic::ceil, OverloadTys: Builder.getDoubleTy());
5673 LogVal = emitNoUnwindRuntimeCall(Builder, Callee: F, Args: LogVal, Name: "");
5674 LogVal = Builder.CreateFPToUI(V: LogVal, DestTy: Builder.getInt32Ty());
5675 llvm::Value *NMin1 = Builder.CreateNUWSub(
5676 LHS: ScanRedInfo->Span,
5677 RHS: llvm::ConstantInt::get(Ty: ScanRedInfo->Span->getType(), V: 1));
5678 Builder.SetInsertPoint(InputBB);
5679 Builder.CreateBr(Dest: LoopBB);
5680 emitBlock(BB: LoopBB, CurFn);
5681 Builder.SetInsertPoint(LoopBB);
5682
5683 PHINode *Counter = Builder.CreatePHI(Ty: Builder.getInt32Ty(), NumReservedValues: 2);
5684 // size pow2k = 1;
5685 PHINode *Pow2K = Builder.CreatePHI(Ty: Builder.getInt32Ty(), NumReservedValues: 2);
5686 Counter->addIncoming(V: llvm::ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0),
5687 BB: InputBB);
5688 Pow2K->addIncoming(V: llvm::ConstantInt::get(Ty: Builder.getInt32Ty(), V: 1),
5689 BB: InputBB);
5690 // for (size i = n - 1; i >= 2 ^ k; --i)
5691 // tmp[i] op= tmp[i-pow2k];
5692 llvm::BasicBlock *InnerLoopBB =
5693 BasicBlock::Create(Context&: CurFn->getContext(), Name: "omp.inner.log.scan.body");
5694 llvm::BasicBlock *InnerExitBB =
5695 BasicBlock::Create(Context&: CurFn->getContext(), Name: "omp.inner.log.scan.exit");
5696 llvm::Value *CmpI = Builder.CreateICmpUGE(LHS: NMin1, RHS: Pow2K);
5697 Builder.CreateCondBr(Cond: CmpI, True: InnerLoopBB, False: InnerExitBB);
5698 emitBlock(BB: InnerLoopBB, CurFn);
5699 Builder.SetInsertPoint(InnerLoopBB);
5700 PHINode *IVal = Builder.CreatePHI(Ty: Builder.getInt32Ty(), NumReservedValues: 2);
5701 IVal->addIncoming(V: NMin1, BB: LoopBB);
5702 for (ReductionInfo RedInfo : ReductionInfos) {
5703 Value *ReductionVal = RedInfo.PrivateVariable;
5704 Value *BuffPtr = (*(ScanRedInfo->ScanBuffPtrs))[ReductionVal];
5705 Value *Buff = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: BuffPtr);
5706 Type *DestTy = RedInfo.ElementType;
5707 Value *IV = Builder.CreateAdd(LHS: IVal, RHS: Builder.getInt32(C: 1));
5708 Value *LHSPtr =
5709 Builder.CreateInBoundsGEP(Ty: DestTy, Ptr: Buff, IdxList: IV, Name: "arrayOffset");
5710 Value *OffsetIval = Builder.CreateNUWSub(LHS: IV, RHS: Pow2K);
5711 Value *RHSPtr =
5712 Builder.CreateInBoundsGEP(Ty: DestTy, Ptr: Buff, IdxList: OffsetIval, Name: "arrayOffset");
5713 Value *LHS = Builder.CreateLoad(Ty: DestTy, Ptr: LHSPtr);
5714 Value *RHS = Builder.CreateLoad(Ty: DestTy, Ptr: RHSPtr);
5715 llvm::Value *Result;
5716 InsertPointOrErrorTy AfterIP =
5717 RedInfo.ReductionGen(Builder.saveIP(), LHS, RHS, Result);
5718 if (!AfterIP)
5719 return AfterIP.takeError();
5720 Builder.CreateStore(Val: Result, Ptr: LHSPtr);
5721 }
5722 llvm::Value *NextIVal = Builder.CreateNUWSub(
5723 LHS: IVal, RHS: llvm::ConstantInt::get(Ty: Builder.getInt32Ty(), V: 1));
5724 IVal->addIncoming(V: NextIVal, BB: Builder.GetInsertBlock());
5725 CmpI = Builder.CreateICmpUGE(LHS: NextIVal, RHS: Pow2K);
5726 Builder.CreateCondBr(Cond: CmpI, True: InnerLoopBB, False: InnerExitBB);
5727 emitBlock(BB: InnerExitBB, CurFn);
5728 llvm::Value *Next = Builder.CreateNUWAdd(
5729 LHS: Counter, RHS: llvm::ConstantInt::get(Ty: Counter->getType(), V: 1));
5730 Counter->addIncoming(V: Next, BB: Builder.GetInsertBlock());
5731 // pow2k <<= 1;
5732 llvm::Value *NextPow2K = Builder.CreateShl(LHS: Pow2K, RHS: 1, Name: "", /*HasNUW=*/true);
5733 Pow2K->addIncoming(V: NextPow2K, BB: Builder.GetInsertBlock());
5734 llvm::Value *Cmp = Builder.CreateICmpNE(LHS: Next, RHS: LogVal);
5735 Builder.CreateCondBr(Cond: Cmp, True: LoopBB, False: ExitBB);
5736 Builder.SetInsertPoint(ExitBB->getFirstInsertionPt());
5737 return Error::success();
5738 };
5739
5740 // TODO: Perform finalization actions for variables. This has to be
5741 // called for variables which have destructors/finalizers.
5742 auto FiniCB = [&](InsertPointTy CodeGenIP) { return llvm::Error::success(); };
5743
5744 llvm::Value *FilterVal = Builder.getInt32(C: 0);
5745 llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
5746 createMasked(Loc: Builder, BodyGenCB, FiniCB, Filter: FilterVal);
5747
5748 if (!AfterIP)
5749 return AfterIP.takeError();
5750 Builder.restoreIP(IP: *AfterIP);
5751 AfterIP = createBarrier(Loc: Builder, Kind: llvm::omp::OMPD_barrier);
5752
5753 if (!AfterIP)
5754 return AfterIP.takeError();
5755 Builder.restoreIP(IP: *AfterIP);
5756 Error Err = emitScanBasedDirectiveFinalsIR(ReductionInfos, ScanRedInfo);
5757 if (Err)
5758 return Err;
5759
5760 return AfterIP;
5761}
5762
5763Error OpenMPIRBuilder::emitScanBasedDirectiveIR(
5764 llvm::function_ref<Error()> InputLoopGen,
5765 llvm::function_ref<Error(LocationDescription Loc)> ScanLoopGen,
5766 ScanInfo *ScanRedInfo) {
5767
5768 {
5769 // Emit loop with input phase:
5770 // for (i: 0..<num_iters>) {
5771 // <input phase>;
5772 // buffer[i] = red;
5773 // }
5774 ScanRedInfo->OMPFirstScanLoop = true;
5775 Error Err = InputLoopGen();
5776 if (Err)
5777 return Err;
5778 }
5779 {
5780 // Emit loop with scan phase:
5781 // for (i: 0..<num_iters>) {
5782 // red = buffer[i];
5783 // <scan phase>;
5784 // }
5785 ScanRedInfo->OMPFirstScanLoop = false;
5786 Error Err = ScanLoopGen(Builder);
5787 if (Err)
5788 return Err;
5789 }
5790 return Error::success();
5791}
5792
5793void OpenMPIRBuilder::createScanBBs(ScanInfo *ScanRedInfo) {
5794 Function *Fun = Builder.GetInsertBlock()->getParent();
5795 ScanRedInfo->OMPScanDispatch =
5796 BasicBlock::Create(Context&: Fun->getContext(), Name: "omp.inscan.dispatch");
5797 ScanRedInfo->OMPAfterScanBlock =
5798 BasicBlock::Create(Context&: Fun->getContext(), Name: "omp.after.scan.bb");
5799 ScanRedInfo->OMPBeforeScanBlock =
5800 BasicBlock::Create(Context&: Fun->getContext(), Name: "omp.before.scan.bb");
5801 ScanRedInfo->OMPScanLoopExit =
5802 BasicBlock::Create(Context&: Fun->getContext(), Name: "omp.scan.loop.exit");
5803}
5804CanonicalLoopInfo *OpenMPIRBuilder::createLoopSkeleton(
5805 DebugLoc DL, Value *TripCount, Function *F, BasicBlock *PreInsertBefore,
5806 BasicBlock *PostInsertBefore, const Twine &Name, bool IsCollapsed) {
5807 Module *M = F->getParent();
5808 LLVMContext &Ctx = M->getContext();
5809 Type *IndVarTy = TripCount->getType();
5810
5811 // Create the basic block structure.
5812 BasicBlock *Preheader =
5813 BasicBlock::Create(Context&: Ctx, Name: "omp_" + Name + ".preheader", Parent: F, InsertBefore: PreInsertBefore);
5814 BasicBlock *Header =
5815 BasicBlock::Create(Context&: Ctx, Name: "omp_" + Name + ".header", Parent: F, InsertBefore: PreInsertBefore);
5816 BasicBlock *Cond =
5817 BasicBlock::Create(Context&: Ctx, Name: "omp_" + Name + ".cond", Parent: F, InsertBefore: PreInsertBefore);
5818 BasicBlock *Body =
5819 BasicBlock::Create(Context&: Ctx, Name: "omp_" + Name + ".body", Parent: F, InsertBefore: PreInsertBefore);
5820 BasicBlock *Latch =
5821 BasicBlock::Create(Context&: Ctx, Name: "omp_" + Name + ".inc", Parent: F, InsertBefore: PostInsertBefore);
5822 BasicBlock *Exit =
5823 BasicBlock::Create(Context&: Ctx, Name: "omp_" + Name + ".exit", Parent: F, InsertBefore: PostInsertBefore);
5824 BasicBlock *After =
5825 BasicBlock::Create(Context&: Ctx, Name: "omp_" + Name + ".after", Parent: F, InsertBefore: PostInsertBefore);
5826
5827 // Use specified DebugLoc for new instructions.
5828 Builder.SetCurrentDebugLocation(DL);
5829
5830 Builder.SetInsertPoint(Preheader);
5831 Builder.CreateBr(Dest: Header);
5832
5833 Builder.SetInsertPoint(Header);
5834 PHINode *IndVarPHI = Builder.CreatePHI(Ty: IndVarTy, NumReservedValues: 2, Name: "omp_" + Name + ".iv");
5835 IndVarPHI->addIncoming(V: ConstantInt::get(Ty: IndVarTy, V: 0), BB: Preheader);
5836 Builder.CreateBr(Dest: Cond);
5837
5838 Builder.SetInsertPoint(Cond);
5839 Value *Cmp =
5840 Builder.CreateICmpULT(LHS: IndVarPHI, RHS: TripCount, Name: "omp_" + Name + ".cmp");
5841 Builder.CreateCondBr(Cond: Cmp, True: Body, False: Exit);
5842
5843 Builder.SetInsertPoint(Body);
5844 Builder.CreateBr(Dest: Latch);
5845
5846 Builder.SetInsertPoint(Latch);
5847 // Decide whether the induction variable increment can carry nsw.
5848 //
5849 // Single loops: nsw is always kept (matching Clang). Any Fortran program
5850 // whose trip count overflows i32 is non-conforming per F2018 11.1.7.4.1, so
5851 // for valid programs 0 <= count <= INT_MAX always holds.
5852 //
5853 // Collapsed loops: the trip count is a product that can overflow i32 even for
5854 // a conforming program, so nsw is kept only when the product is a constant
5855 // that provably fits, dropped otherwise.
5856 bool HasNSW = Config.hasNoSignedWrap();
5857 if (HasNSW) {
5858 if (auto *CI = dyn_cast<ConstantInt>(Val: TripCount)) {
5859 unsigned BitWidth = CI->getType()->getIntegerBitWidth();
5860 APInt SignedMax = APInt::getSignedMaxValue(numBits: BitWidth);
5861 if (CI->getValue().ugt(RHS: SignedMax))
5862 HasNSW = false;
5863 } else if (IsCollapsed) {
5864 HasNSW = false;
5865 }
5866 }
5867 Value *Next =
5868 Builder.CreateAdd(LHS: IndVarPHI, RHS: ConstantInt::get(Ty: IndVarTy, V: 1),
5869 Name: "omp_" + Name + ".next", /*HasNUW=*/true, HasNSW);
5870 Builder.CreateBr(Dest: Header);
5871 IndVarPHI->addIncoming(V: Next, BB: Latch);
5872
5873 Builder.SetInsertPoint(Exit);
5874 Builder.CreateBr(Dest: After);
5875
5876 // Remember and return the canonical control flow.
5877 LoopInfos.emplace_front();
5878 CanonicalLoopInfo *CL = &LoopInfos.front();
5879
5880 CL->Header = Header;
5881 CL->Cond = Cond;
5882 CL->Latch = Latch;
5883 CL->Exit = Exit;
5884
5885#ifndef NDEBUG
5886 CL->assertOK();
5887#endif
5888 return CL;
5889}
5890
5891Expected<CanonicalLoopInfo *>
5892OpenMPIRBuilder::createCanonicalLoop(const LocationDescription &Loc,
5893 LoopBodyGenCallbackTy BodyGenCB,
5894 Value *TripCount, const Twine &Name) {
5895 BasicBlock *BB = Loc.IP.getBlock();
5896 BasicBlock *NextBB = BB->getNextNode();
5897
5898 CanonicalLoopInfo *CL = createLoopSkeleton(DL: Loc.DL, TripCount, F: BB->getParent(),
5899 PreInsertBefore: NextBB, PostInsertBefore: NextBB, Name);
5900 BasicBlock *After = CL->getAfter();
5901
5902 // If location is not set, don't connect the loop.
5903 if (updateToLocation(Loc)) {
5904 // Split the loop at the insertion point: Branch to the preheader and move
5905 // every following instruction to after the loop (the After BB). Also, the
5906 // new successor is the loop's after block.
5907 spliceBB(Builder, New: After, /*CreateBranch=*/false);
5908 Builder.CreateBr(Dest: CL->getPreheader());
5909 }
5910
5911 // Emit the body content. We do it after connecting the loop to the CFG to
5912 // avoid that the callback encounters degenerate BBs.
5913 if (Error Err = BodyGenCB(CL->getBodyIP(), CL->getIndVar()))
5914 return Err;
5915
5916#ifndef NDEBUG
5917 CL->assertOK();
5918#endif
5919 return CL;
5920}
5921
5922Expected<ScanInfo *> OpenMPIRBuilder::scanInfoInitialize() {
5923 ScanInfos.emplace_front();
5924 ScanInfo *Result = &ScanInfos.front();
5925 return Result;
5926}
5927
5928Expected<SmallVector<llvm::CanonicalLoopInfo *>>
5929OpenMPIRBuilder::createCanonicalScanLoops(
5930 const LocationDescription &Loc, LoopBodyGenCallbackTy BodyGenCB,
5931 Value *Start, Value *Stop, Value *Step, bool IsSigned, bool InclusiveStop,
5932 InsertPointTy ComputeIP, const Twine &Name, ScanInfo *ScanRedInfo) {
5933 LocationDescription ComputeLoc =
5934 ComputeIP.isSet() ? LocationDescription(ComputeIP, Loc.DL) : Loc;
5935 updateToLocation(Loc: ComputeLoc);
5936
5937 SmallVector<CanonicalLoopInfo *> Result;
5938
5939 Value *TripCount = calculateCanonicalLoopTripCount(
5940 Loc: ComputeLoc, Start, Stop, Step, IsSigned, InclusiveStop, Name);
5941 ScanRedInfo->Span = TripCount;
5942 ScanRedInfo->OMPScanInit = splitBB(Builder, CreateBranch: true, Name: "scan.init");
5943 Builder.SetInsertPoint(ScanRedInfo->OMPScanInit);
5944
5945 auto BodyGen = [=](InsertPointTy CodeGenIP, Value *IV) {
5946 Builder.restoreIP(IP: CodeGenIP);
5947 ScanRedInfo->IV = IV;
5948 createScanBBs(ScanRedInfo);
5949 BasicBlock *InputBlock = Builder.GetInsertBlock();
5950 Instruction *Terminator = InputBlock->getTerminator();
5951 assert(Terminator->getNumSuccessors() == 1);
5952 BasicBlock *ContinueBlock = Terminator->getSuccessor(Idx: 0);
5953 Terminator->setSuccessor(Idx: 0, BB: ScanRedInfo->OMPScanDispatch);
5954 emitBlock(BB: ScanRedInfo->OMPBeforeScanBlock,
5955 CurFn: Builder.GetInsertBlock()->getParent());
5956 Builder.CreateBr(Dest: ScanRedInfo->OMPScanLoopExit);
5957 emitBlock(BB: ScanRedInfo->OMPScanLoopExit,
5958 CurFn: Builder.GetInsertBlock()->getParent());
5959 Builder.CreateBr(Dest: ContinueBlock);
5960 Builder.SetInsertPoint(
5961 ScanRedInfo->OMPBeforeScanBlock->getFirstInsertionPt());
5962 return BodyGenCB(Builder.saveIP(), IV);
5963 };
5964
5965 const auto &&InputLoopGen = [&]() -> Error {
5966 Expected<CanonicalLoopInfo *> LoopInfo =
5967 createCanonicalLoop(Loc: Builder, BodyGenCB: BodyGen, Start, Stop, Step, IsSigned,
5968 InclusiveStop, ComputeIP, Name, InScan: true, ScanRedInfo);
5969 if (!LoopInfo)
5970 return LoopInfo.takeError();
5971 Result.push_back(Elt: *LoopInfo);
5972 Builder.restoreIP(IP: (*LoopInfo)->getAfterIP());
5973 return Error::success();
5974 };
5975 const auto &&ScanLoopGen = [&](LocationDescription Loc) -> Error {
5976 Expected<CanonicalLoopInfo *> LoopInfo =
5977 createCanonicalLoop(Loc, BodyGenCB: BodyGen, Start, Stop, Step, IsSigned,
5978 InclusiveStop, ComputeIP, Name, InScan: true, ScanRedInfo);
5979 if (!LoopInfo)
5980 return LoopInfo.takeError();
5981 Result.push_back(Elt: *LoopInfo);
5982 Builder.restoreIP(IP: (*LoopInfo)->getAfterIP());
5983 ScanRedInfo->OMPScanFinish = Builder.GetInsertBlock();
5984 return Error::success();
5985 };
5986 Error Err = emitScanBasedDirectiveIR(InputLoopGen, ScanLoopGen, ScanRedInfo);
5987 if (Err)
5988 return Err;
5989 return Result;
5990}
5991
5992Value *OpenMPIRBuilder::calculateCanonicalLoopTripCount(
5993 const LocationDescription &Loc, Value *Start, Value *Stop, Value *Step,
5994 bool IsSigned, bool InclusiveStop, const Twine &Name) {
5995
5996 // Consider the following difficulties (assuming 8-bit signed integers):
5997 // * Adding \p Step to the loop counter which passes \p Stop may overflow:
5998 // DO I = 1, 100, 50
5999 /// * A \p Step of INT_MIN cannot not be normalized to a positive direction:
6000 // DO I = 100, 0, -128
6001
6002 // Start, Stop and Step must be of the same integer type.
6003 auto *IndVarTy = cast<IntegerType>(Val: Start->getType());
6004 assert(IndVarTy == Stop->getType() && "Stop type mismatch");
6005 assert(IndVarTy == Step->getType() && "Step type mismatch");
6006
6007 updateToLocation(Loc);
6008
6009 ConstantInt *Zero = ConstantInt::get(Ty: IndVarTy, V: 0);
6010 ConstantInt *One = ConstantInt::get(Ty: IndVarTy, V: 1);
6011
6012 // Like Step, but always positive.
6013 Value *Incr = Step;
6014
6015 // Distance between Start and Stop; always positive.
6016 Value *Span;
6017
6018 // Condition whether there are no iterations are executed at all, e.g. because
6019 // UB < LB.
6020 Value *ZeroCmp;
6021
6022 if (IsSigned) {
6023 // Ensure that increment is positive. If not, negate and invert LB and UB.
6024 Value *IsNeg = Builder.CreateICmpSLT(LHS: Step, RHS: Zero);
6025 Incr = Builder.CreateSelect(C: IsNeg, True: Builder.CreateNeg(V: Step), False: Step);
6026 Value *LB = Builder.CreateSelect(C: IsNeg, True: Stop, False: Start);
6027 Value *UB = Builder.CreateSelect(C: IsNeg, True: Start, False: Stop);
6028 Span = Builder.CreateSub(LHS: UB, RHS: LB, Name: "", HasNUW: false, HasNSW: true);
6029 ZeroCmp = Builder.CreateICmp(
6030 P: InclusiveStop ? CmpInst::ICMP_SLT : CmpInst::ICMP_SLE, LHS: UB, RHS: LB);
6031 } else {
6032 Span = Builder.CreateSub(LHS: Stop, RHS: Start, Name: "", HasNUW: true);
6033 ZeroCmp = Builder.CreateICmp(
6034 P: InclusiveStop ? CmpInst::ICMP_ULT : CmpInst::ICMP_ULE, LHS: Stop, RHS: Start);
6035 }
6036
6037 Value *CountIfLooping;
6038 if (InclusiveStop) {
6039 CountIfLooping = Builder.CreateAdd(LHS: Builder.CreateUDiv(LHS: Span, RHS: Incr), RHS: One);
6040 } else {
6041 // Avoid incrementing past stop since it could overflow.
6042 Value *CountIfTwo = Builder.CreateAdd(
6043 LHS: Builder.CreateUDiv(LHS: Builder.CreateSub(LHS: Span, RHS: One), RHS: Incr), RHS: One);
6044 Value *OneCmp = Builder.CreateICmp(P: CmpInst::ICMP_ULE, LHS: Span, RHS: Incr);
6045 CountIfLooping = Builder.CreateSelect(C: OneCmp, True: One, False: CountIfTwo);
6046 }
6047
6048 return Builder.CreateSelect(C: ZeroCmp, True: Zero, False: CountIfLooping,
6049 Name: "omp_" + Name + ".tripcount");
6050}
6051
6052Expected<CanonicalLoopInfo *> OpenMPIRBuilder::createCanonicalLoop(
6053 const LocationDescription &Loc, LoopBodyGenCallbackTy BodyGenCB,
6054 Value *Start, Value *Stop, Value *Step, bool IsSigned, bool InclusiveStop,
6055 InsertPointTy ComputeIP, const Twine &Name, bool InScan,
6056 ScanInfo *ScanRedInfo) {
6057 LocationDescription ComputeLoc =
6058 ComputeIP.isSet() ? LocationDescription(ComputeIP, Loc.DL) : Loc;
6059
6060 Value *TripCount = calculateCanonicalLoopTripCount(
6061 Loc: ComputeLoc, Start, Stop, Step, IsSigned, InclusiveStop, Name);
6062
6063 auto BodyGen = [=](InsertPointTy CodeGenIP, Value *IV) {
6064 Builder.restoreIP(IP: CodeGenIP);
6065 Value *Span = Builder.CreateMul(LHS: IV, RHS: Step, Name: "", /*HasNUW=*/false,
6066 /*HasNSW=*/Config.hasNoSignedWrap());
6067 Value *IndVar = Builder.CreateAdd(LHS: Span, RHS: Start, Name: "", /*HasNUW=*/false,
6068 /*HasNSW=*/Config.hasNoSignedWrap());
6069 if (InScan)
6070 ScanRedInfo->IV = IndVar;
6071 return BodyGenCB(Builder.saveIP(), IndVar);
6072 };
6073 LocationDescription LoopLoc =
6074 ComputeIP.isSet()
6075 ? Loc
6076 : LocationDescription(Builder.saveIP(),
6077 Builder.getCurrentDebugLocation());
6078 return createCanonicalLoop(Loc: LoopLoc, BodyGenCB: BodyGen, TripCount, Name);
6079}
6080
6081// Returns an LLVM function to call for initializing loop bounds using OpenMP
6082// static scheduling for composite `distribute parallel for` depending on
6083// `type`. Only i32 and i64 are supported by the runtime. Always interpret
6084// integers as unsigned similarly to CanonicalLoopInfo.
6085static FunctionCallee
6086getKmpcDistForStaticInitForType(Type *Ty, Module &M,
6087 OpenMPIRBuilder &OMPBuilder) {
6088 unsigned Bitwidth = Ty->getIntegerBitWidth();
6089 if (Bitwidth == 32)
6090 return OMPBuilder.getOrCreateRuntimeFunction(
6091 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dist_for_static_init_4u);
6092 if (Bitwidth == 64)
6093 return OMPBuilder.getOrCreateRuntimeFunction(
6094 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dist_for_static_init_8u);
6095 llvm_unreachable("unknown OpenMP loop iterator bitwidth");
6096}
6097
6098// Returns an LLVM function to call for initializing loop bounds using OpenMP
6099// static scheduling depending on `type`. Only i32 and i64 are supported by the
6100// runtime. Always interpret integers as unsigned similarly to
6101// CanonicalLoopInfo.
6102static FunctionCallee getKmpcForStaticInitForType(Type *Ty, Module &M,
6103 OpenMPIRBuilder &OMPBuilder) {
6104 unsigned Bitwidth = Ty->getIntegerBitWidth();
6105 if (Bitwidth == 32)
6106 return OMPBuilder.getOrCreateRuntimeFunction(
6107 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_4u);
6108 if (Bitwidth == 64)
6109 return OMPBuilder.getOrCreateRuntimeFunction(
6110 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_8u);
6111 llvm_unreachable("unknown OpenMP loop iterator bitwidth");
6112}
6113
6114OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::applyStaticWorkshareLoop(
6115 DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP,
6116 WorksharingLoopType LoopType, bool NeedsBarrier, bool HasDistSchedule,
6117 OMPScheduleType DistScheduleSchedType) {
6118 assert(CLI->isValid() && "Requires a valid canonical loop");
6119 assert(!isConflictIP(AllocaIP, CLI->getPreheaderIP()) &&
6120 "Require dedicated allocate IP");
6121
6122 // Set up the source location value for OpenMP runtime.
6123 Builder.restoreIP(IP: CLI->getPreheaderIP());
6124 Builder.SetCurrentDebugLocation(DL);
6125
6126 uint32_t SrcLocStrSize;
6127 Constant *SrcLocStr = getOrCreateSrcLocStr(DL, SrcLocStrSize);
6128 IdentFlag Flag = IdentFlag(0);
6129 switch (LoopType) {
6130 case WorksharingLoopType::ForStaticLoop:
6131 Flag = OMP_IDENT_FLAG_WORK_LOOP;
6132 break;
6133 case WorksharingLoopType::DistributeStaticLoop:
6134 Flag = OMP_IDENT_FLAG_WORK_DISTRIBUTE;
6135 break;
6136 case WorksharingLoopType::DistributeForStaticLoop:
6137 Flag = OMP_IDENT_FLAG_WORK_DISTRIBUTE | OMP_IDENT_FLAG_WORK_LOOP;
6138 break;
6139 }
6140 Value *SrcLoc = getOrCreateIdent(SrcLocStr, SrcLocStrSize, LocFlags: Flag);
6141
6142 // Declare useful OpenMP runtime functions.
6143 Value *IV = CLI->getIndVar();
6144 Type *IVTy = IV->getType();
6145 FunctionCallee StaticInit =
6146 LoopType == WorksharingLoopType::DistributeForStaticLoop
6147 ? getKmpcDistForStaticInitForType(Ty: IVTy, M, OMPBuilder&: *this)
6148 : getKmpcForStaticInitForType(Ty: IVTy, M, OMPBuilder&: *this);
6149 FunctionCallee StaticFini =
6150 getOrCreateRuntimeFunction(M, FnID: omp::OMPRTL___kmpc_for_static_fini);
6151
6152 // Allocate space for computed loop bounds as expected by the "init" function.
6153 Builder.SetInsertPoint(AllocaIP.getBlock()->getFirstNonPHIOrDbgOrAlloca());
6154
6155 Type *I32Type = Type::getInt32Ty(C&: M.getContext());
6156 Value *PLastIter = Builder.CreateAlloca(Ty: I32Type, ArraySize: nullptr, Name: "p.lastiter");
6157 Value *PLowerBound = Builder.CreateAlloca(Ty: IVTy, ArraySize: nullptr, Name: "p.lowerbound");
6158 Value *PUpperBound = Builder.CreateAlloca(Ty: IVTy, ArraySize: nullptr, Name: "p.upperbound");
6159 Value *PStride = Builder.CreateAlloca(Ty: IVTy, ArraySize: nullptr, Name: "p.stride");
6160 CLI->setLastIter(PLastIter);
6161
6162 // At the end of the preheader, prepare for calling the "init" function by
6163 // storing the current loop bounds into the allocated space. A canonical loop
6164 // always iterates from 0 to trip-count with step 1. Note that "init" expects
6165 // and produces an inclusive upper bound.
6166 Builder.SetInsertPoint(CLI->getPreheader()->getTerminator());
6167 Constant *Zero = ConstantInt::get(Ty: IVTy, V: 0);
6168 Constant *One = ConstantInt::get(Ty: IVTy, V: 1);
6169 Builder.CreateStore(Val: Zero, Ptr: PLowerBound);
6170 Value *UpperBound = Builder.CreateSub(LHS: CLI->getTripCount(), RHS: One);
6171 Builder.CreateStore(Val: UpperBound, Ptr: PUpperBound);
6172 Builder.CreateStore(Val: One, Ptr: PStride);
6173
6174 Value *ThreadNum =
6175 getOrCreateThreadID(Ident: getOrCreateIdent(SrcLocStr, SrcLocStrSize));
6176
6177 OMPScheduleType SchedType =
6178 (LoopType == WorksharingLoopType::DistributeStaticLoop)
6179 ? OMPScheduleType::OrderedDistribute
6180 : OMPScheduleType::UnorderedStatic;
6181 Constant *SchedulingType =
6182 ConstantInt::get(Ty: I32Type, V: static_cast<int>(SchedType));
6183
6184 // Call the "init" function and update the trip count of the loop with the
6185 // value it produced.
6186 auto BuildInitCall = [LoopType, SrcLoc, ThreadNum, PLastIter, PLowerBound,
6187 PUpperBound, IVTy, PStride, One, Zero, StaticInit,
6188 this](Value *SchedulingType, auto &Builder) {
6189 SmallVector<Value *, 10> Args({SrcLoc, ThreadNum, SchedulingType, PLastIter,
6190 PLowerBound, PUpperBound});
6191 if (LoopType == WorksharingLoopType::DistributeForStaticLoop) {
6192 Value *PDistUpperBound =
6193 Builder.CreateAlloca(IVTy, nullptr, "p.distupperbound");
6194 Args.push_back(Elt: PDistUpperBound);
6195 }
6196 Args.append(IL: {PStride, One, Zero});
6197 createRuntimeFunctionCall(Callee: StaticInit, Args);
6198 };
6199 BuildInitCall(SchedulingType, Builder);
6200 if (HasDistSchedule &&
6201 LoopType != WorksharingLoopType::DistributeStaticLoop) {
6202 Constant *DistScheduleSchedType = ConstantInt::get(
6203 Ty: I32Type, V: static_cast<int>(omp::OMPScheduleType::OrderedDistribute));
6204 // We want to emit a second init function call for the dist_schedule clause
6205 // to the Distribute construct. This should only be done however if a
6206 // Workshare Loop is nested within a Distribute Construct
6207 BuildInitCall(DistScheduleSchedType, Builder);
6208 }
6209 Value *LowerBound = Builder.CreateLoad(Ty: IVTy, Ptr: PLowerBound);
6210 Value *InclusiveUpperBound = Builder.CreateLoad(Ty: IVTy, Ptr: PUpperBound);
6211 Value *TripCountMinusOne = Builder.CreateSub(LHS: InclusiveUpperBound, RHS: LowerBound);
6212 Value *TripCount = Builder.CreateAdd(LHS: TripCountMinusOne, RHS: One);
6213 CLI->setTripCount(TripCount);
6214
6215 // Update all uses of the induction variable except the one in the condition
6216 // block that compares it with the actual upper bound, and the increment in
6217 // the latch block.
6218
6219 CLI->mapIndVar(Updater: [&](Instruction *OldIV) -> Value * {
6220 Builder.SetInsertPoint(TheBB: CLI->getBody(),
6221 IP: CLI->getBody()->getFirstInsertionPt());
6222 Builder.SetCurrentDebugLocation(DL);
6223 return Builder.CreateAdd(LHS: OldIV, RHS: LowerBound, Name: "", /*HasNUW=*/false,
6224 /*HasNSW=*/Config.hasNoSignedWrap());
6225 });
6226
6227 // In the "exit" block, call the "fini" function.
6228 Builder.SetInsertPoint(TheBB: CLI->getExit(),
6229 IP: CLI->getExit()->getTerminator()->getIterator());
6230 createRuntimeFunctionCall(Callee: StaticFini, Args: {SrcLoc, ThreadNum});
6231
6232 // Add the barrier if requested.
6233 if (NeedsBarrier) {
6234 InsertPointOrErrorTy BarrierIP =
6235 createBarrier(Loc: LocationDescription(Builder.saveIP(), DL),
6236 Kind: omp::Directive::OMPD_for, /* ForceSimpleCall */ false,
6237 /* CheckCancelFlag */ false);
6238 if (!BarrierIP)
6239 return BarrierIP.takeError();
6240 }
6241
6242 InsertPointTy AfterIP = CLI->getAfterIP();
6243 CLI->invalidate();
6244
6245 return AfterIP;
6246}
6247
6248static void addAccessGroupMetadata(BasicBlock *Block, MDNode *AccessGroup,
6249 LoopInfo &LI);
6250static void addLoopMetadata(CanonicalLoopInfo *Loop,
6251 ArrayRef<Metadata *> Properties);
6252
6253static void applyParallelAccessesMetadata(CanonicalLoopInfo *CLI,
6254 LLVMContext &Ctx, Loop *Loop,
6255 LoopInfo &LoopInfo,
6256 SmallVector<Metadata *> &LoopMDList) {
6257 SmallSet<BasicBlock *, 8> Reachable;
6258
6259 // Get the basic blocks from the loop in which memref instructions
6260 // can be found.
6261 // TODO: Generalize getting all blocks inside a CanonicalizeLoopInfo,
6262 // preferably without running any passes.
6263 for (BasicBlock *Block : Loop->getBlocks()) {
6264 if (Block == CLI->getCond() || Block == CLI->getHeader())
6265 continue;
6266 Reachable.insert(Ptr: Block);
6267 }
6268
6269 // Add access group metadata to memory-access instructions.
6270 MDNode *AccessGroup = MDNode::getDistinct(Context&: Ctx, MDs: {});
6271 for (BasicBlock *BB : Reachable)
6272 addAccessGroupMetadata(Block: BB, AccessGroup, LI&: LoopInfo);
6273 // TODO: If the loop has existing parallel access metadata, have
6274 // to combine two lists.
6275 LoopMDList.push_back(Elt: MDNode::get(
6276 Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.parallel_accesses"), AccessGroup}));
6277}
6278
6279OpenMPIRBuilder::InsertPointOrErrorTy
6280OpenMPIRBuilder::applyStaticChunkedWorkshareLoop(
6281 DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP,
6282 bool NeedsBarrier, Value *ChunkSize, OMPScheduleType SchedType,
6283 Value *DistScheduleChunkSize, OMPScheduleType DistScheduleSchedType) {
6284 assert(CLI->isValid() && "Requires a valid canonical loop");
6285 assert((ChunkSize || DistScheduleChunkSize) && "Chunk size is required");
6286
6287 LLVMContext &Ctx = CLI->getFunction()->getContext();
6288 Value *IV = CLI->getIndVar();
6289 Value *OrigTripCount = CLI->getTripCount();
6290 Type *IVTy = IV->getType();
6291 assert(IVTy->getIntegerBitWidth() <= 64 &&
6292 "Max supported tripcount bitwidth is 64 bits");
6293 Type *InternalIVTy = IVTy->getIntegerBitWidth() <= 32 ? Type::getInt32Ty(C&: Ctx)
6294 : Type::getInt64Ty(C&: Ctx);
6295 Type *I32Type = Type::getInt32Ty(C&: M.getContext());
6296 Constant *Zero = ConstantInt::get(Ty: InternalIVTy, V: 0);
6297 Constant *One = ConstantInt::get(Ty: InternalIVTy, V: 1);
6298
6299 Function *F = CLI->getFunction();
6300 // Blocks must have terminators.
6301 // FIXME: Don't run analyses on incomplete/invalid IR.
6302 SmallVector<Instruction *> UIs;
6303 for (BasicBlock &BB : *F)
6304 if (!BB.hasTerminator())
6305 UIs.push_back(Elt: new UnreachableInst(F->getContext(), &BB));
6306 FunctionAnalysisManager FAM;
6307 FAM.registerPass(PassBuilder: []() { return DominatorTreeAnalysis(); });
6308 FAM.registerPass(PassBuilder: []() { return PassInstrumentationAnalysis(); });
6309 LoopAnalysis LIA;
6310 LoopInfo &&LI = LIA.run(F&: *F, AM&: FAM);
6311 for (Instruction *I : UIs)
6312 I->eraseFromParent();
6313 Loop *L = LI.getLoopFor(BB: CLI->getHeader());
6314 SmallVector<Metadata *> LoopMDList;
6315 if (ChunkSize || DistScheduleChunkSize)
6316 applyParallelAccessesMetadata(CLI, Ctx, Loop: L, LoopInfo&: LI, LoopMDList);
6317 addLoopMetadata(Loop: CLI, Properties: LoopMDList);
6318
6319 // Declare useful OpenMP runtime functions.
6320 FunctionCallee StaticInit =
6321 getKmpcForStaticInitForType(Ty: InternalIVTy, M, OMPBuilder&: *this);
6322 FunctionCallee StaticFini =
6323 getOrCreateRuntimeFunction(M, FnID: omp::OMPRTL___kmpc_for_static_fini);
6324
6325 // Allocate space for computed loop bounds as expected by the "init" function.
6326 Builder.restoreIP(IP: AllocaIP);
6327 Builder.SetCurrentDebugLocation(DL);
6328 Value *PLastIter = Builder.CreateAlloca(Ty: I32Type, ArraySize: nullptr, Name: "p.lastiter");
6329 Value *PLowerBound =
6330 Builder.CreateAlloca(Ty: InternalIVTy, ArraySize: nullptr, Name: "p.lowerbound");
6331 Value *PUpperBound =
6332 Builder.CreateAlloca(Ty: InternalIVTy, ArraySize: nullptr, Name: "p.upperbound");
6333 Value *PStride = Builder.CreateAlloca(Ty: InternalIVTy, ArraySize: nullptr, Name: "p.stride");
6334 CLI->setLastIter(PLastIter);
6335
6336 // Set up the source location value for the OpenMP runtime.
6337 Builder.restoreIP(IP: CLI->getPreheaderIP());
6338 Builder.SetCurrentDebugLocation(DL);
6339
6340 // TODO: Detect overflow in ubsan or max-out with current tripcount.
6341 Value *CastedChunkSize = Builder.CreateZExtOrTrunc(
6342 V: ChunkSize ? ChunkSize : Zero, DestTy: InternalIVTy, Name: "chunksize");
6343 Value *CastedDistScheduleChunkSize = Builder.CreateZExtOrTrunc(
6344 V: DistScheduleChunkSize ? DistScheduleChunkSize : Zero, DestTy: InternalIVTy,
6345 Name: "distschedulechunksize");
6346 Value *CastedTripCount =
6347 Builder.CreateZExt(V: OrigTripCount, DestTy: InternalIVTy, Name: "tripcount");
6348
6349 Constant *SchedulingType =
6350 ConstantInt::get(Ty: I32Type, V: static_cast<int>(SchedType));
6351 Constant *DistSchedulingType =
6352 ConstantInt::get(Ty: I32Type, V: static_cast<int>(DistScheduleSchedType));
6353 Builder.CreateStore(Val: Zero, Ptr: PLowerBound);
6354 Value *OrigUpperBound = Builder.CreateSub(LHS: CastedTripCount, RHS: One);
6355 Value *IsTripCountZero = Builder.CreateICmpEQ(LHS: CastedTripCount, RHS: Zero);
6356 Value *UpperBound =
6357 Builder.CreateSelect(C: IsTripCountZero, True: Zero, False: OrigUpperBound);
6358 Builder.CreateStore(Val: UpperBound, Ptr: PUpperBound);
6359 Builder.CreateStore(Val: One, Ptr: PStride);
6360
6361 // Call the "init" function and update the trip count of the loop with the
6362 // value it produced.
6363 uint32_t SrcLocStrSize;
6364 Constant *SrcLocStr = getOrCreateSrcLocStr(DL, SrcLocStrSize);
6365 IdentFlag Flag = OMP_IDENT_FLAG_WORK_LOOP;
6366 if (DistScheduleSchedType != OMPScheduleType::None) {
6367 Flag |= OMP_IDENT_FLAG_WORK_DISTRIBUTE;
6368 }
6369 Value *SrcLoc = getOrCreateIdent(SrcLocStr, SrcLocStrSize, LocFlags: Flag);
6370 Value *ThreadNum =
6371 getOrCreateThreadID(Ident: getOrCreateIdent(SrcLocStr, SrcLocStrSize));
6372 auto BuildInitCall = [StaticInit, SrcLoc, ThreadNum, PLastIter, PLowerBound,
6373 PUpperBound, PStride, One,
6374 this](Value *SchedulingType, Value *ChunkSize,
6375 auto &Builder) {
6376 createRuntimeFunctionCall(
6377 Callee: StaticInit, Args: {/*loc=*/SrcLoc, /*global_tid=*/ThreadNum,
6378 /*schedtype=*/SchedulingType, /*plastiter=*/PLastIter,
6379 /*plower=*/PLowerBound, /*pupper=*/PUpperBound,
6380 /*pstride=*/PStride, /*incr=*/One,
6381 /*chunk=*/ChunkSize});
6382 };
6383 BuildInitCall(SchedulingType, CastedChunkSize, Builder);
6384 if (DistScheduleSchedType != OMPScheduleType::None &&
6385 SchedType != OMPScheduleType::OrderedDistributeChunked &&
6386 SchedType != OMPScheduleType::OrderedDistribute) {
6387 // We want to emit a second init function call for the dist_schedule clause
6388 // to the Distribute construct. This should only be done however if a
6389 // Workshare Loop is nested within a Distribute Construct
6390 BuildInitCall(DistSchedulingType, CastedDistScheduleChunkSize, Builder);
6391 }
6392
6393 // Load values written by the "init" function.
6394 Value *FirstChunkStart =
6395 Builder.CreateLoad(Ty: InternalIVTy, Ptr: PLowerBound, Name: "omp_firstchunk.lb");
6396 Value *FirstChunkStop =
6397 Builder.CreateLoad(Ty: InternalIVTy, Ptr: PUpperBound, Name: "omp_firstchunk.ub");
6398 Value *FirstChunkEnd = Builder.CreateAdd(LHS: FirstChunkStop, RHS: One);
6399 Value *ChunkRange =
6400 Builder.CreateSub(LHS: FirstChunkEnd, RHS: FirstChunkStart, Name: "omp_chunk.range");
6401 Value *NextChunkStride =
6402 Builder.CreateLoad(Ty: InternalIVTy, Ptr: PStride, Name: "omp_dispatch.stride");
6403
6404 // Create outer "dispatch" loop for enumerating the chunks.
6405 BasicBlock *DispatchEnter = splitBB(Builder, CreateBranch: true);
6406 Value *DispatchCounter;
6407
6408 // It is safe to assume this didn't return an error because the callback
6409 // passed into createCanonicalLoop is the only possible error source, and it
6410 // always returns success.
6411 CanonicalLoopInfo *DispatchCLI = cantFail(ValOrErr: createCanonicalLoop(
6412 Loc: {Builder.saveIP(), DL},
6413 BodyGenCB: [&](InsertPointTy BodyIP, Value *Counter) {
6414 DispatchCounter = Counter;
6415 return Error::success();
6416 },
6417 Start: FirstChunkStart, Stop: CastedTripCount, Step: NextChunkStride,
6418 /*IsSigned=*/false, /*InclusiveStop=*/false, /*ComputeIP=*/{},
6419 Name: "dispatch"));
6420
6421 // Remember the BasicBlocks of the dispatch loop we need, then invalidate to
6422 // not have to preserve the canonical invariant.
6423 BasicBlock *DispatchBody = DispatchCLI->getBody();
6424 BasicBlock *DispatchLatch = DispatchCLI->getLatch();
6425 BasicBlock *DispatchExit = DispatchCLI->getExit();
6426 BasicBlock *DispatchAfter = DispatchCLI->getAfter();
6427 DispatchCLI->invalidate();
6428
6429 // Rewire the original loop to become the chunk loop inside the dispatch loop.
6430 redirectTo(Source: DispatchAfter, Target: CLI->getAfter(), DL);
6431 redirectTo(Source: CLI->getExit(), Target: DispatchLatch, DL);
6432 redirectTo(Source: DispatchBody, Target: DispatchEnter, DL);
6433
6434 // Prepare the prolog of the chunk loop.
6435 Builder.restoreIP(IP: CLI->getPreheaderIP());
6436 Builder.SetCurrentDebugLocation(DL);
6437
6438 // Compute the number of iterations of the chunk loop.
6439 Builder.SetInsertPoint(CLI->getPreheader()->getTerminator());
6440 Value *ChunkEnd = Builder.CreateAdd(LHS: DispatchCounter, RHS: ChunkRange);
6441 Value *IsLastChunk =
6442 Builder.CreateICmpUGE(LHS: ChunkEnd, RHS: CastedTripCount, Name: "omp_chunk.is_last");
6443 Value *CountUntilOrigTripCount =
6444 Builder.CreateSub(LHS: CastedTripCount, RHS: DispatchCounter);
6445 Value *ChunkTripCount = Builder.CreateSelect(
6446 C: IsLastChunk, True: CountUntilOrigTripCount, False: ChunkRange, Name: "omp_chunk.tripcount");
6447 Value *BackcastedChunkTC =
6448 Builder.CreateTrunc(V: ChunkTripCount, DestTy: IVTy, Name: "omp_chunk.tripcount.trunc");
6449 CLI->setTripCount(BackcastedChunkTC);
6450
6451 // Update all uses of the induction variable except the one in the condition
6452 // block that compares it with the actual upper bound, and the increment in
6453 // the latch block.
6454 Value *BackcastedDispatchCounter =
6455 Builder.CreateTrunc(V: DispatchCounter, DestTy: IVTy, Name: "omp_dispatch.iv.trunc");
6456 CLI->mapIndVar(Updater: [&](Instruction *) -> Value * {
6457 Builder.restoreIP(IP: CLI->getBodyIP());
6458 return Builder.CreateAdd(LHS: IV, RHS: BackcastedDispatchCounter);
6459 });
6460
6461 // In the "exit" block, call the "fini" function.
6462 Builder.SetInsertPoint(TheBB: DispatchExit, IP: DispatchExit->getFirstInsertionPt());
6463 createRuntimeFunctionCall(Callee: StaticFini, Args: {SrcLoc, ThreadNum});
6464
6465 // Add the barrier if requested.
6466 if (NeedsBarrier) {
6467 InsertPointOrErrorTy AfterIP =
6468 createBarrier(Loc: LocationDescription(Builder.saveIP(), DL), Kind: OMPD_for,
6469 /*ForceSimpleCall=*/false, /*CheckCancelFlag=*/false);
6470 if (!AfterIP)
6471 return AfterIP.takeError();
6472 }
6473
6474#ifndef NDEBUG
6475 // Even though we currently do not support applying additional methods to it,
6476 // the chunk loop should remain a canonical loop.
6477 CLI->assertOK();
6478#endif
6479
6480 return InsertPointTy(DispatchAfter, DispatchAfter->getFirstInsertionPt());
6481}
6482
6483// Returns an LLVM function to call for executing an OpenMP static worksharing
6484// for loop depending on `type`. Only i32 and i64 are supported by the runtime.
6485// Always interpret integers as unsigned similarly to CanonicalLoopInfo.
6486static FunctionCallee
6487getKmpcForStaticLoopForType(Type *Ty, OpenMPIRBuilder *OMPBuilder,
6488 WorksharingLoopType LoopType) {
6489 unsigned Bitwidth = Ty->getIntegerBitWidth();
6490 Module &M = OMPBuilder->M;
6491 switch (LoopType) {
6492 case WorksharingLoopType::ForStaticLoop:
6493 if (Bitwidth == 32)
6494 return OMPBuilder->getOrCreateRuntimeFunction(
6495 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_for_static_loop_4u);
6496 if (Bitwidth == 64)
6497 return OMPBuilder->getOrCreateRuntimeFunction(
6498 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_for_static_loop_8u);
6499 break;
6500 case WorksharingLoopType::DistributeStaticLoop:
6501 if (Bitwidth == 32)
6502 return OMPBuilder->getOrCreateRuntimeFunction(
6503 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_distribute_static_loop_4u);
6504 if (Bitwidth == 64)
6505 return OMPBuilder->getOrCreateRuntimeFunction(
6506 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_distribute_static_loop_8u);
6507 break;
6508 case WorksharingLoopType::DistributeForStaticLoop:
6509 if (Bitwidth == 32)
6510 return OMPBuilder->getOrCreateRuntimeFunction(
6511 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_distribute_for_static_loop_4u);
6512 if (Bitwidth == 64)
6513 return OMPBuilder->getOrCreateRuntimeFunction(
6514 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_distribute_for_static_loop_8u);
6515 break;
6516 }
6517 if (Bitwidth != 32 && Bitwidth != 64) {
6518 llvm_unreachable("Unknown OpenMP loop iterator bitwidth");
6519 }
6520 llvm_unreachable("Unknown type of OpenMP worksharing loop");
6521}
6522
6523// Inserts a call to proper OpenMP Device RTL function which handles
6524// loop worksharing.
6525static void createTargetLoopWorkshareCall(OpenMPIRBuilder *OMPBuilder,
6526 WorksharingLoopType LoopType,
6527 BasicBlock *InsertBlock, Value *Ident,
6528 Value *LoopBodyArg, Value *TripCount,
6529 Function &LoopBodyFn, bool NoLoop) {
6530 Type *TripCountTy = TripCount->getType();
6531 Module &M = OMPBuilder->M;
6532 IRBuilder<> &Builder = OMPBuilder->Builder;
6533 FunctionCallee RTLFn =
6534 getKmpcForStaticLoopForType(Ty: TripCountTy, OMPBuilder, LoopType);
6535 SmallVector<Value *, 8> RealArgs;
6536 RealArgs.push_back(Elt: Ident);
6537 RealArgs.push_back(Elt: &LoopBodyFn);
6538 RealArgs.push_back(Elt: LoopBodyArg);
6539 RealArgs.push_back(Elt: TripCount);
6540 if (LoopType == WorksharingLoopType::DistributeStaticLoop) {
6541 RealArgs.push_back(Elt: ConstantInt::get(Ty: TripCountTy, V: 0));
6542 RealArgs.push_back(Elt: ConstantInt::get(Ty: Builder.getInt8Ty(), V: 0));
6543 Builder.restoreIP(IP: {InsertBlock, std::prev(x: InsertBlock->end())});
6544 OMPBuilder->createRuntimeFunctionCall(Callee: RTLFn, Args: RealArgs);
6545 return;
6546 }
6547 FunctionCallee RTLNumThreads = OMPBuilder->getOrCreateRuntimeFunction(
6548 M, FnID: omp::RuntimeFunction::OMPRTL_omp_get_num_threads);
6549 Builder.restoreIP(IP: {InsertBlock, std::prev(x: InsertBlock->end())});
6550 Value *NumThreads = OMPBuilder->createRuntimeFunctionCall(Callee: RTLNumThreads, Args: {});
6551
6552 RealArgs.push_back(
6553 Elt: Builder.CreateZExtOrTrunc(V: NumThreads, DestTy: TripCountTy, Name: "num.threads.cast"));
6554 RealArgs.push_back(Elt: ConstantInt::get(Ty: TripCountTy, V: 0));
6555 if (LoopType == WorksharingLoopType::DistributeForStaticLoop) {
6556 RealArgs.push_back(Elt: ConstantInt::get(Ty: TripCountTy, V: 0));
6557 RealArgs.push_back(Elt: ConstantInt::get(Ty: Builder.getInt8Ty(), V: NoLoop));
6558 } else {
6559 RealArgs.push_back(Elt: ConstantInt::get(Ty: Builder.getInt8Ty(), V: 0));
6560 }
6561
6562 OMPBuilder->createRuntimeFunctionCall(Callee: RTLFn, Args: RealArgs);
6563}
6564
6565static void workshareLoopTargetCallback(
6566 OpenMPIRBuilder *OMPIRBuilder, CanonicalLoopInfo *CLI, Value *Ident,
6567 Function &OutlinedFn, const SmallVector<Instruction *, 4> &ToBeDeleted,
6568 WorksharingLoopType LoopType, bool NoLoop) {
6569 IRBuilder<> &Builder = OMPIRBuilder->Builder;
6570 BasicBlock *Preheader = CLI->getPreheader();
6571 Value *TripCount = CLI->getTripCount();
6572
6573 // After loop body outling, the loop body contains only set up
6574 // of loop body argument structure and the call to the outlined
6575 // loop body function. Firstly, we need to move setup of loop body args
6576 // into loop preheader.
6577 Preheader->splice(ToIt: std::prev(x: Preheader->end()), FromBB: CLI->getBody(),
6578 FromBeginIt: CLI->getBody()->begin(), FromEndIt: std::prev(x: CLI->getBody()->end()));
6579
6580 // The next step is to remove the whole loop. We do not it need anymore.
6581 // That's why make an unconditional branch from loop preheader to loop
6582 // exit block
6583 Builder.restoreIP(IP: {Preheader, Preheader->end()});
6584 Builder.SetCurrentDebugLocation(Preheader->getTerminator()->getDebugLoc());
6585 Preheader->getTerminator()->eraseFromParent();
6586 Builder.CreateBr(Dest: CLI->getExit());
6587
6588 // Delete dead loop blocks
6589 OpenMPIRBuilder::OutlineInfo CleanUpInfo;
6590 SmallPtrSet<BasicBlock *, 32> RegionBlockSet;
6591 SmallVector<BasicBlock *, 32> BlocksToBeRemoved;
6592 CleanUpInfo.EntryBB = CLI->getHeader();
6593 CleanUpInfo.ExitBB = CLI->getExit();
6594 CleanUpInfo.collectBlocks(BlockSet&: RegionBlockSet, BlockVector&: BlocksToBeRemoved);
6595 DeleteDeadBlocks(BBs: BlocksToBeRemoved);
6596
6597 // Find the instruction which corresponds to loop body argument structure
6598 // and remove the call to loop body function instruction.
6599 Value *LoopBodyArg;
6600 User *OutlinedFnUser = OutlinedFn.getUniqueUndroppableUser();
6601 assert(OutlinedFnUser &&
6602 "Expected unique undroppable user of outlined function");
6603 CallInst *OutlinedFnCallInstruction = dyn_cast<CallInst>(Val: OutlinedFnUser);
6604 assert(OutlinedFnCallInstruction && "Expected outlined function call");
6605 assert((OutlinedFnCallInstruction->getParent() == Preheader) &&
6606 "Expected outlined function call to be located in loop preheader");
6607 // Check in case no argument structure has been passed.
6608 if (OutlinedFnCallInstruction->arg_size() > 1)
6609 LoopBodyArg = OutlinedFnCallInstruction->getArgOperand(i: 1);
6610 else
6611 LoopBodyArg = Constant::getNullValue(Ty: Builder.getPtrTy());
6612 OutlinedFnCallInstruction->eraseFromParent();
6613
6614 createTargetLoopWorkshareCall(OMPBuilder: OMPIRBuilder, LoopType, InsertBlock: Preheader, Ident,
6615 LoopBodyArg, TripCount, LoopBodyFn&: OutlinedFn, NoLoop);
6616
6617 for (auto &ToBeDeletedItem : ToBeDeleted)
6618 ToBeDeletedItem->eraseFromParent();
6619 CLI->invalidate();
6620}
6621
6622OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::applyWorkshareLoopTarget(
6623 DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP,
6624 WorksharingLoopType LoopType, bool NoLoop) {
6625 uint32_t SrcLocStrSize;
6626 Constant *SrcLocStr = getOrCreateSrcLocStr(DL, SrcLocStrSize);
6627 IdentFlag Flag = IdentFlag(0);
6628 switch (LoopType) {
6629 case WorksharingLoopType::ForStaticLoop:
6630 Flag = OMP_IDENT_FLAG_WORK_LOOP;
6631 break;
6632 case WorksharingLoopType::DistributeStaticLoop:
6633 Flag = OMP_IDENT_FLAG_WORK_DISTRIBUTE;
6634 break;
6635 case WorksharingLoopType::DistributeForStaticLoop:
6636 Flag = OMP_IDENT_FLAG_WORK_DISTRIBUTE | OMP_IDENT_FLAG_WORK_LOOP;
6637 break;
6638 }
6639 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize, LocFlags: Flag);
6640
6641 auto OI = std::make_unique<OutlineInfo>();
6642 OI->OuterAllocBB = CLI->getPreheader();
6643 Function *OuterFn = CLI->getPreheader()->getParent();
6644
6645 // Instructions which need to be deleted at the end of code generation
6646 SmallVector<Instruction *, 4> ToBeDeleted;
6647
6648 OI->OuterAllocBB = AllocaIP.getBlock();
6649
6650 // Mark the body loop as region which needs to be extracted
6651 OI->EntryBB = CLI->getBody();
6652 OI->ExitBB = CLI->getLatch()->splitBasicBlockBefore(I: CLI->getLatch()->begin(),
6653 BBName: "omp.prelatch");
6654
6655 // Prepare loop body for extraction
6656 Builder.restoreIP(IP: {CLI->getPreheader(), CLI->getPreheader()->begin()});
6657
6658 // Insert new loop counter variable which will be used only in loop
6659 // body.
6660 AllocaInst *NewLoopCnt = Builder.CreateAlloca(Ty: CLI->getIndVarType(), ArraySize: 0, Name: "");
6661 Instruction *NewLoopCntLoad =
6662 Builder.CreateLoad(Ty: CLI->getIndVarType(), Ptr: NewLoopCnt);
6663 // New loop counter instructions are redundant in the loop preheader when
6664 // code generation for workshare loop is finshed. That's why mark them as
6665 // ready for deletion.
6666 ToBeDeleted.push_back(Elt: NewLoopCntLoad);
6667 ToBeDeleted.push_back(Elt: NewLoopCnt);
6668
6669 // Analyse loop body region. Find all input variables which are used inside
6670 // loop body region.
6671 SmallPtrSet<BasicBlock *, 32> ParallelRegionBlockSet;
6672 SmallVector<BasicBlock *, 32> Blocks;
6673 OI->collectBlocks(BlockSet&: ParallelRegionBlockSet, BlockVector&: Blocks);
6674
6675 CodeExtractorAnalysisCache CEAC(*OuterFn);
6676 CodeExtractor Extractor(Blocks,
6677 /* DominatorTree */ nullptr,
6678 /* AggregateArgs */ true,
6679 /* BlockFrequencyInfo */ nullptr,
6680 /* BranchProbabilityInfo */ nullptr,
6681 /* AssumptionCache */ nullptr,
6682 /* AllowVarArgs */ true,
6683 /* AllowAlloca */ true,
6684 /* AllocationBlock */ CLI->getPreheader(),
6685 /* DeallocationBlocks */ {},
6686 /* Suffix */ ".omp_wsloop",
6687 /* AggrArgsIn0AddrSpace */ true);
6688
6689 BasicBlock *CommonExit = nullptr;
6690 SetVector<Value *> SinkingCands, HoistingCands;
6691
6692 // Find allocas outside the loop body region which are used inside loop
6693 // body
6694 Extractor.findAllocas(CEAC, SinkCands&: SinkingCands, HoistCands&: HoistingCands, ExitBlock&: CommonExit);
6695
6696 // We need to model loop body region as the function f(cnt, loop_arg).
6697 // That's why we replace loop induction variable by the new counter
6698 // which will be one of loop body function argument
6699 SmallVector<User *> Users(CLI->getIndVar()->user_begin(),
6700 CLI->getIndVar()->user_end());
6701 for (auto Use : Users) {
6702 if (Instruction *Inst = dyn_cast<Instruction>(Val: Use)) {
6703 if (ParallelRegionBlockSet.count(Ptr: Inst->getParent())) {
6704 Inst->replaceUsesOfWith(From: CLI->getIndVar(), To: NewLoopCntLoad);
6705 }
6706 }
6707 }
6708 // Make sure that loop counter variable is not merged into loop body
6709 // function argument structure and it is passed as separate variable
6710 OI->ExcludeArgsFromAggregate.push_back(Elt: NewLoopCntLoad);
6711
6712 // PostOutline CB is invoked when loop body function is outlined and
6713 // loop body is replaced by call to outlined function. We need to add
6714 // call to OpenMP device rtl inside loop preheader. OpenMP device rtl
6715 // function will handle loop control logic.
6716 //
6717 OI->PostOutlineCB = [=, ToBeDeletedVec =
6718 std::move(ToBeDeleted)](Function &OutlinedFn) {
6719 workshareLoopTargetCallback(OMPIRBuilder: this, CLI, Ident, OutlinedFn, ToBeDeleted: ToBeDeletedVec,
6720 LoopType, NoLoop);
6721 };
6722 addOutlineInfo(OI: std::move(OI));
6723 return CLI->getAfterIP();
6724}
6725
6726OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::applyWorkshareLoop(
6727 DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP,
6728 bool NeedsBarrier, omp::ScheduleKind SchedKind, Value *ChunkSize,
6729 bool HasSimdModifier, bool HasMonotonicModifier,
6730 bool HasNonmonotonicModifier, bool HasOrderedClause,
6731 WorksharingLoopType LoopType, bool NoLoop, bool HasDistSchedule,
6732 Value *DistScheduleChunkSize) {
6733 if (Config.isTargetDevice())
6734 return applyWorkshareLoopTarget(DL, CLI, AllocaIP, LoopType, NoLoop);
6735 OMPScheduleType EffectiveScheduleType = computeOpenMPScheduleType(
6736 ClauseKind: SchedKind, HasChunks: ChunkSize, HasSimdModifier, HasMonotonicModifier,
6737 HasNonmonotonicModifier, HasOrderedClause, HasDistScheduleChunks: DistScheduleChunkSize);
6738
6739 bool IsOrdered = (EffectiveScheduleType & OMPScheduleType::ModifierOrdered) ==
6740 OMPScheduleType::ModifierOrdered;
6741 OMPScheduleType DistScheduleSchedType = OMPScheduleType::None;
6742 if (HasDistSchedule) {
6743 DistScheduleSchedType = DistScheduleChunkSize
6744 ? OMPScheduleType::OrderedDistributeChunked
6745 : OMPScheduleType::OrderedDistribute;
6746 }
6747 switch (EffectiveScheduleType & ~OMPScheduleType::ModifierMask) {
6748 case OMPScheduleType::BaseStatic:
6749 case OMPScheduleType::BaseDistribute:
6750 assert((!ChunkSize || !DistScheduleChunkSize) &&
6751 "No chunk size with static-chunked schedule");
6752 if (IsOrdered && !HasDistSchedule)
6753 return applyDynamicWorkshareLoop(DL, CLI, AllocaIP, SchedType: EffectiveScheduleType,
6754 NeedsBarrier, Chunk: ChunkSize);
6755 // FIXME: Monotonicity ignored?
6756 if (DistScheduleChunkSize)
6757 return applyStaticChunkedWorkshareLoop(
6758 DL, CLI, AllocaIP, NeedsBarrier, ChunkSize, SchedType: EffectiveScheduleType,
6759 DistScheduleChunkSize, DistScheduleSchedType);
6760 return applyStaticWorkshareLoop(DL, CLI, AllocaIP, LoopType, NeedsBarrier,
6761 HasDistSchedule);
6762
6763 case OMPScheduleType::BaseStaticChunked:
6764 case OMPScheduleType::BaseDistributeChunked:
6765 if (IsOrdered && !HasDistSchedule)
6766 return applyDynamicWorkshareLoop(DL, CLI, AllocaIP, SchedType: EffectiveScheduleType,
6767 NeedsBarrier, Chunk: ChunkSize);
6768 // FIXME: Monotonicity ignored?
6769 return applyStaticChunkedWorkshareLoop(
6770 DL, CLI, AllocaIP, NeedsBarrier, ChunkSize, SchedType: EffectiveScheduleType,
6771 DistScheduleChunkSize, DistScheduleSchedType);
6772
6773 case OMPScheduleType::BaseRuntime:
6774 case OMPScheduleType::BaseAuto:
6775 case OMPScheduleType::BaseGreedy:
6776 case OMPScheduleType::BaseBalanced:
6777 case OMPScheduleType::BaseSteal:
6778 case OMPScheduleType::BaseRuntimeSimd:
6779 assert(!ChunkSize &&
6780 "schedule type does not support user-defined chunk sizes");
6781 [[fallthrough]];
6782 case OMPScheduleType::BaseGuidedSimd:
6783 case OMPScheduleType::BaseDynamicChunked:
6784 case OMPScheduleType::BaseGuidedChunked:
6785 case OMPScheduleType::BaseGuidedIterativeChunked:
6786 case OMPScheduleType::BaseGuidedAnalyticalChunked:
6787 case OMPScheduleType::BaseStaticBalancedChunked:
6788 return applyDynamicWorkshareLoop(DL, CLI, AllocaIP, SchedType: EffectiveScheduleType,
6789 NeedsBarrier, Chunk: ChunkSize);
6790
6791 default:
6792 llvm_unreachable("Unknown/unimplemented schedule kind");
6793 }
6794}
6795
6796/// Returns an LLVM function to call for initializing loop bounds using OpenMP
6797/// dynamic scheduling depending on `type`. Only i32 and i64 are supported by
6798/// the runtime. Always interpret integers as unsigned similarly to
6799/// CanonicalLoopInfo.
6800static FunctionCallee
6801getKmpcForDynamicInitForType(Type *Ty, Module &M, OpenMPIRBuilder &OMPBuilder) {
6802 unsigned Bitwidth = Ty->getIntegerBitWidth();
6803 if (Bitwidth == 32)
6804 return OMPBuilder.getOrCreateRuntimeFunction(
6805 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dispatch_init_4u);
6806 if (Bitwidth == 64)
6807 return OMPBuilder.getOrCreateRuntimeFunction(
6808 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dispatch_init_8u);
6809 llvm_unreachable("unknown OpenMP loop iterator bitwidth");
6810}
6811
6812/// Returns an LLVM function to call for updating the next loop using OpenMP
6813/// dynamic scheduling depending on `type`. Only i32 and i64 are supported by
6814/// the runtime. Always interpret integers as unsigned similarly to
6815/// CanonicalLoopInfo.
6816static FunctionCallee
6817getKmpcForDynamicNextForType(Type *Ty, Module &M, OpenMPIRBuilder &OMPBuilder) {
6818 unsigned Bitwidth = Ty->getIntegerBitWidth();
6819 if (Bitwidth == 32)
6820 return OMPBuilder.getOrCreateRuntimeFunction(
6821 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dispatch_next_4u);
6822 if (Bitwidth == 64)
6823 return OMPBuilder.getOrCreateRuntimeFunction(
6824 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dispatch_next_8u);
6825 llvm_unreachable("unknown OpenMP loop iterator bitwidth");
6826}
6827
6828/// Returns an LLVM function to call for finalizing the dynamic loop using
6829/// depending on `type`. Only i32 and i64 are supported by the runtime. Always
6830/// interpret integers as unsigned similarly to CanonicalLoopInfo.
6831static FunctionCallee
6832getKmpcForDynamicFiniForType(Type *Ty, Module &M, OpenMPIRBuilder &OMPBuilder) {
6833 unsigned Bitwidth = Ty->getIntegerBitWidth();
6834 if (Bitwidth == 32)
6835 return OMPBuilder.getOrCreateRuntimeFunction(
6836 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dispatch_fini_4u);
6837 if (Bitwidth == 64)
6838 return OMPBuilder.getOrCreateRuntimeFunction(
6839 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dispatch_fini_8u);
6840 llvm_unreachable("unknown OpenMP loop iterator bitwidth");
6841}
6842
6843OpenMPIRBuilder::InsertPointOrErrorTy
6844OpenMPIRBuilder::applyDynamicWorkshareLoop(DebugLoc DL, CanonicalLoopInfo *CLI,
6845 InsertPointTy AllocaIP,
6846 OMPScheduleType SchedType,
6847 bool NeedsBarrier, Value *Chunk) {
6848 assert(CLI->isValid() && "Requires a valid canonical loop");
6849 assert(!isConflictIP(AllocaIP, CLI->getPreheaderIP()) &&
6850 "Require dedicated allocate IP");
6851 assert(isValidWorkshareLoopScheduleType(SchedType) &&
6852 "Require valid schedule type");
6853
6854 bool Ordered = (SchedType & OMPScheduleType::ModifierOrdered) ==
6855 OMPScheduleType::ModifierOrdered;
6856
6857 // Set up the source location value for OpenMP runtime.
6858 Builder.SetCurrentDebugLocation(DL);
6859
6860 uint32_t SrcLocStrSize;
6861 Constant *SrcLocStr = getOrCreateSrcLocStr(DL, SrcLocStrSize);
6862 Value *SrcLoc =
6863 getOrCreateIdent(SrcLocStr, SrcLocStrSize, LocFlags: OMP_IDENT_FLAG_WORK_LOOP);
6864
6865 // Declare useful OpenMP runtime functions.
6866 Value *IV = CLI->getIndVar();
6867 Type *IVTy = IV->getType();
6868 FunctionCallee DynamicInit = getKmpcForDynamicInitForType(Ty: IVTy, M, OMPBuilder&: *this);
6869 FunctionCallee DynamicNext = getKmpcForDynamicNextForType(Ty: IVTy, M, OMPBuilder&: *this);
6870
6871 // Allocate space for computed loop bounds as expected by the "init" function.
6872 Builder.SetInsertPoint(AllocaIP.getBlock()->getFirstNonPHIOrDbgOrAlloca());
6873 Type *I32Type = Type::getInt32Ty(C&: M.getContext());
6874 Value *PLastIter = Builder.CreateAlloca(Ty: I32Type, ArraySize: nullptr, Name: "p.lastiter");
6875 Value *PLowerBound = Builder.CreateAlloca(Ty: IVTy, ArraySize: nullptr, Name: "p.lowerbound");
6876 Value *PUpperBound = Builder.CreateAlloca(Ty: IVTy, ArraySize: nullptr, Name: "p.upperbound");
6877 Value *PStride = Builder.CreateAlloca(Ty: IVTy, ArraySize: nullptr, Name: "p.stride");
6878 CLI->setLastIter(PLastIter);
6879
6880 // At the end of the preheader, prepare for calling the "init" function by
6881 // storing the current loop bounds into the allocated space. A canonical loop
6882 // always iterates from 0 to trip-count with step 1. Note that "init" expects
6883 // and produces an inclusive upper bound.
6884 BasicBlock *PreHeader = CLI->getPreheader();
6885 Builder.SetInsertPoint(PreHeader->getTerminator());
6886 Constant *One = ConstantInt::get(Ty: IVTy, V: 1);
6887 Builder.CreateStore(Val: One, Ptr: PLowerBound);
6888 Value *UpperBound = CLI->getTripCount();
6889 Builder.CreateStore(Val: UpperBound, Ptr: PUpperBound);
6890 Builder.CreateStore(Val: One, Ptr: PStride);
6891
6892 BasicBlock *Header = CLI->getHeader();
6893 BasicBlock *Exit = CLI->getExit();
6894 BasicBlock *Cond = CLI->getCond();
6895 BasicBlock *Latch = CLI->getLatch();
6896 InsertPointTy AfterIP = CLI->getAfterIP();
6897
6898 // The CLI will be "broken" in the code below, as the loop is no longer
6899 // a valid canonical loop.
6900
6901 if (!Chunk)
6902 Chunk = One;
6903
6904 Value *ThreadNum =
6905 getOrCreateThreadID(Ident: getOrCreateIdent(SrcLocStr, SrcLocStrSize));
6906
6907 Constant *SchedulingType =
6908 ConstantInt::get(Ty: I32Type, V: static_cast<int>(SchedType));
6909
6910 // Call the "init" function.
6911 createRuntimeFunctionCall(Callee: DynamicInit, Args: {SrcLoc, ThreadNum, SchedulingType,
6912 /* LowerBound */ One, UpperBound,
6913 /* step */ One, Chunk});
6914
6915 // An outer loop around the existing one.
6916 BasicBlock *OuterCond = BasicBlock::Create(
6917 Context&: PreHeader->getContext(), Name: Twine(PreHeader->getName()) + ".outer.cond",
6918 Parent: PreHeader->getParent());
6919 // This needs to be 32-bit always, so can't use the IVTy Zero above.
6920 Builder.SetInsertPoint(TheBB: OuterCond, IP: OuterCond->getFirstInsertionPt());
6921 Value *Res = createRuntimeFunctionCall(
6922 Callee: DynamicNext,
6923 Args: {SrcLoc, ThreadNum, PLastIter, PLowerBound, PUpperBound, PStride});
6924 Constant *Zero32 = ConstantInt::get(Ty: I32Type, V: 0);
6925 Value *MoreWork = Builder.CreateCmp(Pred: CmpInst::ICMP_NE, LHS: Res, RHS: Zero32);
6926 Value *LowerBound =
6927 Builder.CreateSub(LHS: Builder.CreateLoad(Ty: IVTy, Ptr: PLowerBound), RHS: One, Name: "lb");
6928 Builder.CreateCondBr(Cond: MoreWork, True: Header, False: Exit);
6929
6930 // Change PHI-node in loop header to use outer cond rather than preheader,
6931 // and set IV to the LowerBound.
6932 Instruction *Phi = &Header->front();
6933 auto *PI = cast<PHINode>(Val: Phi);
6934 PI->setIncomingBlock(i: 0, BB: OuterCond);
6935 PI->setIncomingValue(i: 0, V: LowerBound);
6936
6937 // Then set the pre-header to jump to the OuterCond
6938 Instruction *Term = PreHeader->getTerminator();
6939 auto *Br = cast<UncondBrInst>(Val: Term);
6940 Br->setSuccessor(OuterCond);
6941
6942 // Modify the inner condition:
6943 // * Use the UpperBound returned from the DynamicNext call.
6944 // * jump to the loop outer loop when done with one of the inner loops.
6945 Builder.SetInsertPoint(TheBB: Cond, IP: Cond->getFirstInsertionPt());
6946 UpperBound = Builder.CreateLoad(Ty: IVTy, Ptr: PUpperBound, Name: "ub");
6947 Instruction *Comp = &*Builder.GetInsertPoint();
6948 auto *CI = cast<CmpInst>(Val: Comp);
6949 CI->setOperand(i_nocapture: 1, Val_nocapture: UpperBound);
6950 // Redirect the inner exit to branch to outer condition.
6951 Instruction *Branch = &Cond->back();
6952 auto *BI = cast<CondBrInst>(Val: Branch);
6953 assert(BI->getSuccessor(1) == Exit);
6954 BI->setSuccessor(idx: 1, NewSucc: OuterCond);
6955
6956 // Call the "fini" function if "ordered" is present in wsloop directive.
6957 if (Ordered) {
6958 Builder.SetInsertPoint(&Latch->back());
6959 FunctionCallee DynamicFini = getKmpcForDynamicFiniForType(Ty: IVTy, M, OMPBuilder&: *this);
6960 createRuntimeFunctionCall(Callee: DynamicFini, Args: {SrcLoc, ThreadNum});
6961 }
6962
6963 // Add the barrier if requested.
6964 if (NeedsBarrier) {
6965 Builder.SetInsertPoint(&Exit->back());
6966 InsertPointOrErrorTy BarrierIP =
6967 createBarrier(Loc: LocationDescription(Builder.saveIP(), DL),
6968 Kind: omp::Directive::OMPD_for, /* ForceSimpleCall */ false,
6969 /* CheckCancelFlag */ false);
6970 if (!BarrierIP)
6971 return BarrierIP.takeError();
6972 }
6973
6974 CLI->invalidate();
6975 return AfterIP;
6976}
6977
6978/// Redirect all edges that branch to \p OldTarget to \p NewTarget. That is,
6979/// after this \p OldTarget will be orphaned.
6980static void redirectAllPredecessorsTo(BasicBlock *OldTarget,
6981 BasicBlock *NewTarget, DebugLoc DL) {
6982 for (BasicBlock *Pred : make_early_inc_range(Range: predecessors(BB: OldTarget)))
6983 redirectTo(Source: Pred, Target: NewTarget, DL);
6984}
6985
6986static void removeUnusedBlocksFromParent(ArrayRef<BasicBlock *> BBs) {
6987 SmallPtrSet<BasicBlock *, 8> InternalBBs(from_range, BBs);
6988 // We add a block to BBsToKeep iff we have proven it has an external use.
6989 SmallPtrSet<BasicBlock *, 8> BBsToKeep;
6990
6991 while (true) {
6992 bool Changed = false;
6993
6994 for (BasicBlock *BB : BBs) {
6995 if (BBsToKeep.contains(Ptr: BB))
6996 continue;
6997
6998 for (Use &U : BB->uses()) {
6999 auto *UseInst = dyn_cast<Instruction>(Val: U.getUser());
7000 if (!UseInst)
7001 continue;
7002 BasicBlock *UseBB = UseInst->getParent();
7003 if (!InternalBBs.contains(Ptr: UseBB) || BBsToKeep.contains(Ptr: UseBB)) {
7004 BBsToKeep.insert(Ptr: BB);
7005 Changed = true;
7006 break;
7007 }
7008 }
7009 }
7010
7011 if (!Changed)
7012 break;
7013 }
7014
7015 SmallVector<BasicBlock *> BBsToDelete = filter_to_vector(
7016 C&: BBs, Pred: [&BBsToKeep](BasicBlock *BB) { return !BBsToKeep.contains(Ptr: BB); });
7017 DeleteDeadBlocks(BBs: BBsToDelete);
7018}
7019
7020CanonicalLoopInfo *
7021OpenMPIRBuilder::collapseLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
7022 InsertPointTy ComputeIP) {
7023 assert(Loops.size() >= 1 && "At least one loop required");
7024 size_t NumLoops = Loops.size();
7025
7026 // Nothing to do if there is already just one loop.
7027 if (NumLoops == 1)
7028 return Loops.front();
7029
7030 CanonicalLoopInfo *Outermost = Loops.front();
7031 CanonicalLoopInfo *Innermost = Loops.back();
7032 BasicBlock *OrigPreheader = Outermost->getPreheader();
7033 BasicBlock *OrigAfter = Outermost->getAfter();
7034 Function *F = OrigPreheader->getParent();
7035
7036 // Loop control blocks that may become orphaned later.
7037 SmallVector<BasicBlock *, 12> OldControlBBs;
7038 OldControlBBs.reserve(N: 6 * Loops.size());
7039 for (CanonicalLoopInfo *Loop : Loops)
7040 Loop->collectControlBlocks(BBs&: OldControlBBs);
7041
7042 // Setup the IRBuilder for inserting the trip count computation.
7043 Builder.SetCurrentDebugLocation(DL);
7044 if (ComputeIP.isSet())
7045 Builder.restoreIP(IP: ComputeIP);
7046 else
7047 Builder.restoreIP(IP: Outermost->getPreheaderIP());
7048
7049 // Derive the collapsed' loop trip count.
7050 // TODO: Find common/largest indvar type.
7051 Value *CollapsedTripCount = nullptr;
7052 for (CanonicalLoopInfo *L : Loops) {
7053 assert(L->isValid() &&
7054 "All loops to collapse must be valid canonical loops");
7055 Value *OrigTripCount = L->getTripCount();
7056 if (!CollapsedTripCount) {
7057 CollapsedTripCount = OrigTripCount;
7058 continue;
7059 }
7060
7061 // TODO: Enable UndefinedSanitizer to diagnose an overflow here.
7062 CollapsedTripCount =
7063 Builder.CreateNUWMul(LHS: CollapsedTripCount, RHS: OrigTripCount);
7064 }
7065
7066 // Create the collapsed loop control flow.
7067 CanonicalLoopInfo *Result =
7068 createLoopSkeleton(DL, TripCount: CollapsedTripCount, F,
7069 PreInsertBefore: OrigPreheader->getNextNode(), PostInsertBefore: OrigAfter, Name: "collapsed",
7070 /*IsCollapsed=*/true);
7071
7072 // Build the collapsed loop body code.
7073 // Start with deriving the input loop induction variables from the collapsed
7074 // one, using a divmod scheme. To preserve the original loops' order, the
7075 // innermost loop use the least significant bits.
7076 Builder.restoreIP(IP: Result->getBodyIP());
7077
7078 Value *Leftover = Result->getIndVar();
7079 SmallVector<Value *> NewIndVars;
7080 NewIndVars.resize(N: NumLoops);
7081 for (int i = NumLoops - 1; i >= 1; --i) {
7082 Value *OrigTripCount = Loops[i]->getTripCount();
7083
7084 Value *NewIndVar = Builder.CreateURem(LHS: Leftover, RHS: OrigTripCount);
7085 NewIndVars[i] = NewIndVar;
7086
7087 Leftover = Builder.CreateUDiv(LHS: Leftover, RHS: OrigTripCount);
7088 }
7089 // Outermost loop gets all the remaining bits.
7090 NewIndVars[0] = Leftover;
7091
7092 // Construct the loop body control flow.
7093 // We progressively construct the branch structure following in direction of
7094 // the control flow, from the leading in-between code, the loop nest body, the
7095 // trailing in-between code, and rejoining the collapsed loop's latch.
7096 // ContinueBlock and ContinuePred keep track of the source(s) of next edge. If
7097 // the ContinueBlock is set, continue with that block. If ContinuePred, use
7098 // its predecessors as sources.
7099 BasicBlock *ContinueBlock = Result->getBody();
7100 BasicBlock *ContinuePred = nullptr;
7101 auto ContinueWith = [&ContinueBlock, &ContinuePred, DL](BasicBlock *Dest,
7102 BasicBlock *NextSrc) {
7103 if (ContinueBlock)
7104 redirectTo(Source: ContinueBlock, Target: Dest, DL);
7105 else
7106 redirectAllPredecessorsTo(OldTarget: ContinuePred, NewTarget: Dest, DL);
7107
7108 ContinueBlock = nullptr;
7109 ContinuePred = NextSrc;
7110 };
7111
7112 // The code before the nested loop of each level.
7113 // Because we are sinking it into the nest, it will be executed more often
7114 // that the original loop. More sophisticated schemes could keep track of what
7115 // the in-between code is and instantiate it only once per thread.
7116 for (size_t i = 0; i < NumLoops - 1; ++i)
7117 ContinueWith(Loops[i]->getBody(), Loops[i + 1]->getHeader());
7118
7119 // Connect the loop nest body.
7120 ContinueWith(Innermost->getBody(), Innermost->getLatch());
7121
7122 // The code after the nested loop at each level.
7123 for (size_t i = NumLoops - 1; i > 0; --i)
7124 ContinueWith(Loops[i]->getAfter(), Loops[i - 1]->getLatch());
7125
7126 // Connect the finished loop to the collapsed loop latch.
7127 ContinueWith(Result->getLatch(), nullptr);
7128
7129 // Replace the input loops with the new collapsed loop.
7130 redirectTo(Source: Outermost->getPreheader(), Target: Result->getPreheader(), DL);
7131 redirectTo(Source: Result->getAfter(), Target: Outermost->getAfter(), DL);
7132
7133 // Replace the input loop indvars with the derived ones.
7134 for (size_t i = 0; i < NumLoops; ++i)
7135 Loops[i]->getIndVar()->replaceAllUsesWith(V: NewIndVars[i]);
7136
7137 // Remove unused parts of the input loops.
7138 removeUnusedBlocksFromParent(BBs: OldControlBBs);
7139
7140 for (CanonicalLoopInfo *L : Loops)
7141 L->invalidate();
7142
7143#ifndef NDEBUG
7144 Result->assertOK();
7145#endif
7146 return Result;
7147}
7148
7149std::vector<CanonicalLoopInfo *>
7150OpenMPIRBuilder::tileLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
7151 ArrayRef<Value *> TileSizes) {
7152 assert(TileSizes.size() == Loops.size() &&
7153 "Must pass as many tile sizes as there are loops");
7154 int NumLoops = Loops.size();
7155 assert(NumLoops >= 1 && "At least one loop to tile required");
7156
7157 CanonicalLoopInfo *OutermostLoop = Loops.front();
7158 CanonicalLoopInfo *InnermostLoop = Loops.back();
7159 Function *F = OutermostLoop->getBody()->getParent();
7160 BasicBlock *InnerEnter = InnermostLoop->getBody();
7161 BasicBlock *InnerLatch = InnermostLoop->getLatch();
7162
7163 // Loop control blocks that may become orphaned later.
7164 SmallVector<BasicBlock *, 12> OldControlBBs;
7165 OldControlBBs.reserve(N: 6 * Loops.size());
7166 for (CanonicalLoopInfo *Loop : Loops)
7167 Loop->collectControlBlocks(BBs&: OldControlBBs);
7168
7169 // Collect original trip counts and induction variable to be accessible by
7170 // index. Also, the structure of the original loops is not preserved during
7171 // the construction of the tiled loops, so do it before we scavenge the BBs of
7172 // any original CanonicalLoopInfo.
7173 SmallVector<Value *, 4> OrigTripCounts, OrigIndVars;
7174 for (CanonicalLoopInfo *L : Loops) {
7175 assert(L->isValid() && "All input loops must be valid canonical loops");
7176 OrigTripCounts.push_back(Elt: L->getTripCount());
7177 OrigIndVars.push_back(Elt: L->getIndVar());
7178 }
7179
7180 // Collect the code between loop headers. These may contain SSA definitions
7181 // that are used in the loop nest body. To be usable with in the innermost
7182 // body, these BasicBlocks will be sunk into the loop nest body. That is,
7183 // these instructions may be executed more often than before the tiling.
7184 // TODO: It would be sufficient to only sink them into body of the
7185 // corresponding tile loop.
7186 SmallVector<std::pair<BasicBlock *, BasicBlock *>, 4> InbetweenCode;
7187 for (int i = 0; i < NumLoops - 1; ++i) {
7188 CanonicalLoopInfo *Surrounding = Loops[i];
7189 CanonicalLoopInfo *Nested = Loops[i + 1];
7190
7191 BasicBlock *EnterBB = Surrounding->getBody();
7192 BasicBlock *ExitBB = Nested->getHeader();
7193 InbetweenCode.emplace_back(Args&: EnterBB, Args&: ExitBB);
7194 }
7195
7196 // Compute the trip counts of the floor loops.
7197 Builder.SetCurrentDebugLocation(DL);
7198 Builder.restoreIP(IP: OutermostLoop->getPreheaderIP());
7199 SmallVector<Value *, 4> FloorCompleteCount, FloorCount, FloorRems;
7200 for (int i = 0; i < NumLoops; ++i) {
7201 Value *TileSize = TileSizes[i];
7202 Value *OrigTripCount = OrigTripCounts[i];
7203 Type *IVType = OrigTripCount->getType();
7204
7205 Value *FloorCompleteTripCount = Builder.CreateUDiv(LHS: OrigTripCount, RHS: TileSize);
7206 Value *FloorTripRem = Builder.CreateURem(LHS: OrigTripCount, RHS: TileSize);
7207
7208 // 0 if tripcount divides the tilesize, 1 otherwise.
7209 // 1 means we need an additional iteration for a partial tile.
7210 //
7211 // Unfortunately we cannot just use the roundup-formula
7212 // (tripcount + tilesize - 1)/tilesize
7213 // because the summation might overflow. We do not want introduce undefined
7214 // behavior when the untiled loop nest did not.
7215 Value *FloorTripOverflow =
7216 Builder.CreateICmpNE(LHS: FloorTripRem, RHS: ConstantInt::get(Ty: IVType, V: 0));
7217
7218 FloorTripOverflow = Builder.CreateZExt(V: FloorTripOverflow, DestTy: IVType);
7219 Value *FloorTripCount =
7220 Builder.CreateAdd(LHS: FloorCompleteTripCount, RHS: FloorTripOverflow,
7221 Name: "omp_floor" + Twine(i) + ".tripcount", HasNUW: true);
7222
7223 // Remember some values for later use.
7224 FloorCompleteCount.push_back(Elt: FloorCompleteTripCount);
7225 FloorCount.push_back(Elt: FloorTripCount);
7226 FloorRems.push_back(Elt: FloorTripRem);
7227 }
7228
7229 // Generate the new loop nest, from the outermost to the innermost.
7230 std::vector<CanonicalLoopInfo *> Result;
7231 Result.reserve(n: NumLoops * 2);
7232
7233 // The basic block of the surrounding loop that enters the nest generated
7234 // loop.
7235 BasicBlock *Enter = OutermostLoop->getPreheader();
7236
7237 // The basic block of the surrounding loop where the inner code should
7238 // continue.
7239 BasicBlock *Continue = OutermostLoop->getAfter();
7240
7241 // Where the next loop basic block should be inserted.
7242 BasicBlock *OutroInsertBefore = InnermostLoop->getExit();
7243
7244 auto EmbeddNewLoop =
7245 [this, DL, F, InnerEnter, &Enter, &Continue, &OutroInsertBefore](
7246 Value *TripCount, const Twine &Name) -> CanonicalLoopInfo * {
7247 CanonicalLoopInfo *EmbeddedLoop = createLoopSkeleton(
7248 DL, TripCount, F, PreInsertBefore: InnerEnter, PostInsertBefore: OutroInsertBefore, Name);
7249 redirectTo(Source: Enter, Target: EmbeddedLoop->getPreheader(), DL);
7250 redirectTo(Source: EmbeddedLoop->getAfter(), Target: Continue, DL);
7251
7252 // Setup the position where the next embedded loop connects to this loop.
7253 Enter = EmbeddedLoop->getBody();
7254 Continue = EmbeddedLoop->getLatch();
7255 OutroInsertBefore = EmbeddedLoop->getLatch();
7256 return EmbeddedLoop;
7257 };
7258
7259 auto EmbeddNewLoops = [&Result, &EmbeddNewLoop](ArrayRef<Value *> TripCounts,
7260 const Twine &NameBase) {
7261 for (auto P : enumerate(First&: TripCounts)) {
7262 CanonicalLoopInfo *EmbeddedLoop =
7263 EmbeddNewLoop(P.value(), NameBase + Twine(P.index()));
7264 Result.push_back(x: EmbeddedLoop);
7265 }
7266 };
7267
7268 EmbeddNewLoops(FloorCount, "floor");
7269
7270 // Within the innermost floor loop, emit the code that computes the tile
7271 // sizes.
7272 Builder.SetInsertPoint(Enter->getTerminator());
7273 SmallVector<Value *, 4> TileCounts;
7274 for (int i = 0; i < NumLoops; ++i) {
7275 CanonicalLoopInfo *FloorLoop = Result[i];
7276 Value *TileSize = TileSizes[i];
7277
7278 Value *FloorIsEpilogue =
7279 Builder.CreateICmpEQ(LHS: FloorLoop->getIndVar(), RHS: FloorCompleteCount[i]);
7280 Value *TileTripCount =
7281 Builder.CreateSelect(C: FloorIsEpilogue, True: FloorRems[i], False: TileSize);
7282
7283 TileCounts.push_back(Elt: TileTripCount);
7284 }
7285
7286 // Create the tile loops.
7287 EmbeddNewLoops(TileCounts, "tile");
7288
7289 // Insert the inbetween code into the body.
7290 BasicBlock *BodyEnter = Enter;
7291 BasicBlock *BodyEntered = nullptr;
7292 for (std::pair<BasicBlock *, BasicBlock *> P : InbetweenCode) {
7293 BasicBlock *EnterBB = P.first;
7294 BasicBlock *ExitBB = P.second;
7295
7296 if (BodyEnter)
7297 redirectTo(Source: BodyEnter, Target: EnterBB, DL);
7298 else
7299 redirectAllPredecessorsTo(OldTarget: BodyEntered, NewTarget: EnterBB, DL);
7300
7301 BodyEnter = nullptr;
7302 BodyEntered = ExitBB;
7303 }
7304
7305 // Append the original loop nest body into the generated loop nest body.
7306 if (BodyEnter)
7307 redirectTo(Source: BodyEnter, Target: InnerEnter, DL);
7308 else
7309 redirectAllPredecessorsTo(OldTarget: BodyEntered, NewTarget: InnerEnter, DL);
7310 redirectAllPredecessorsTo(OldTarget: InnerLatch, NewTarget: Continue, DL);
7311
7312 // Replace the original induction variable with an induction variable computed
7313 // from the tile and floor induction variables.
7314 Builder.restoreIP(IP: Result.back()->getBodyIP());
7315 for (int i = 0; i < NumLoops; ++i) {
7316 CanonicalLoopInfo *FloorLoop = Result[i];
7317 CanonicalLoopInfo *TileLoop = Result[NumLoops + i];
7318 Value *OrigIndVar = OrigIndVars[i];
7319 Value *Size = TileSizes[i];
7320
7321 Value *Scale =
7322 Builder.CreateMul(LHS: Size, RHS: FloorLoop->getIndVar(), Name: {}, /*HasNUW=*/true);
7323 Value *Shift =
7324 Builder.CreateAdd(LHS: Scale, RHS: TileLoop->getIndVar(), Name: {}, /*HasNUW=*/true);
7325 OrigIndVar->replaceAllUsesWith(V: Shift);
7326 }
7327
7328 // Remove unused parts of the original loops.
7329 removeUnusedBlocksFromParent(BBs: OldControlBBs);
7330
7331 for (CanonicalLoopInfo *L : Loops)
7332 L->invalidate();
7333
7334#ifndef NDEBUG
7335 for (CanonicalLoopInfo *GenL : Result)
7336 GenL->assertOK();
7337#endif
7338 return Result;
7339}
7340
7341/// Attach metadata \p Properties to the basic block described by \p BB. If the
7342/// basic block already has metadata, the basic block properties are appended.
7343static void addBasicBlockMetadata(BasicBlock *BB,
7344 ArrayRef<Metadata *> Properties) {
7345 // Nothing to do if no property to attach.
7346 if (Properties.empty())
7347 return;
7348
7349 LLVMContext &Ctx = BB->getContext();
7350 SmallVector<Metadata *> NewProperties;
7351 NewProperties.push_back(Elt: nullptr);
7352
7353 // If the basic block already has metadata, prepend it to the new metadata.
7354 MDNode *Existing = BB->getTerminator()->getMetadata(KindID: LLVMContext::MD_loop);
7355 if (Existing)
7356 append_range(C&: NewProperties, R: drop_begin(RangeOrContainer: Existing->operands(), N: 1));
7357
7358 append_range(C&: NewProperties, R&: Properties);
7359 MDNode *BasicBlockID = MDNode::getDistinct(Context&: Ctx, MDs: NewProperties);
7360 BasicBlockID->replaceOperandWith(I: 0, New: BasicBlockID);
7361
7362 BB->getTerminator()->setMetadata(KindID: LLVMContext::MD_loop, Node: BasicBlockID);
7363}
7364
7365/// Attach loop metadata \p Properties to the loop described by \p Loop. If the
7366/// loop already has metadata, the loop properties are appended.
7367static void addLoopMetadata(CanonicalLoopInfo *Loop,
7368 ArrayRef<Metadata *> Properties) {
7369 assert(Loop->isValid() && "Expecting a valid CanonicalLoopInfo");
7370
7371 // Attach metadata to the loop's latch
7372 BasicBlock *Latch = Loop->getLatch();
7373 assert(Latch && "A valid CanonicalLoopInfo must have a unique latch");
7374 addBasicBlockMetadata(BB: Latch, Properties);
7375}
7376
7377/// Attach llvm.access.group metadata to the memref instructions of \p Block
7378static void addAccessGroupMetadata(BasicBlock *Block, MDNode *AccessGroup,
7379 LoopInfo &LI) {
7380 for (Instruction &I : *Block) {
7381 if (I.mayReadOrWriteMemory()) {
7382 // TODO: This instruction may already have access group from
7383 // other pragmas e.g. #pragma clang loop vectorize. Append
7384 // so that the existing metadata is not overwritten.
7385 I.setMetadata(KindID: LLVMContext::MD_access_group, Node: AccessGroup);
7386 }
7387 }
7388}
7389
7390CanonicalLoopInfo *
7391OpenMPIRBuilder::fuseLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops) {
7392 CanonicalLoopInfo *firstLoop = Loops.front();
7393 CanonicalLoopInfo *lastLoop = Loops.back();
7394 Function *F = firstLoop->getPreheader()->getParent();
7395
7396 // Loop control blocks that will become orphaned later
7397 SmallVector<BasicBlock *> oldControlBBs;
7398 for (CanonicalLoopInfo *Loop : Loops)
7399 Loop->collectControlBlocks(BBs&: oldControlBBs);
7400
7401 // Collect original trip counts
7402 SmallVector<Value *> origTripCounts;
7403 for (CanonicalLoopInfo *L : Loops) {
7404 assert(L->isValid() && "All input loops must be valid canonical loops");
7405 origTripCounts.push_back(Elt: L->getTripCount());
7406 }
7407
7408 Builder.SetCurrentDebugLocation(DL);
7409
7410 // Compute max trip count.
7411 // The fused loop will be from 0 to max(origTripCounts)
7412 BasicBlock *TCBlock = BasicBlock::Create(Context&: F->getContext(), Name: "omp.fuse.comp.tc",
7413 Parent: F, InsertBefore: firstLoop->getHeader());
7414 Builder.SetInsertPoint(TCBlock);
7415 Value *fusedTripCount = nullptr;
7416 for (CanonicalLoopInfo *L : Loops) {
7417 assert(L->isValid() && "All loops to fuse must be valid canonical loops");
7418 Value *origTripCount = L->getTripCount();
7419 if (!fusedTripCount) {
7420 fusedTripCount = origTripCount;
7421 continue;
7422 }
7423 Value *condTP = Builder.CreateICmpSGT(LHS: fusedTripCount, RHS: origTripCount);
7424 fusedTripCount = Builder.CreateSelect(C: condTP, True: fusedTripCount, False: origTripCount,
7425 Name: ".omp.fuse.tc");
7426 }
7427
7428 // Generate new loop
7429 CanonicalLoopInfo *fused =
7430 createLoopSkeleton(DL, TripCount: fusedTripCount, F, PreInsertBefore: firstLoop->getBody(),
7431 PostInsertBefore: lastLoop->getLatch(), Name: "fused");
7432
7433 // Replace original loops with the fused loop
7434 // Preheader and After are not considered inside the CLI.
7435 // These are used to compute the individual TCs of the loops
7436 // so they have to be put before the resulting fused loop.
7437 // Moving them up for readability.
7438 for (size_t i = 0; i < Loops.size() - 1; ++i) {
7439 Loops[i]->getPreheader()->moveBefore(MovePos: TCBlock);
7440 Loops[i]->getAfter()->moveBefore(MovePos: TCBlock);
7441 }
7442 lastLoop->getPreheader()->moveBefore(MovePos: TCBlock);
7443
7444 for (size_t i = 0; i < Loops.size() - 1; ++i) {
7445 redirectTo(Source: Loops[i]->getPreheader(), Target: Loops[i]->getAfter(), DL);
7446 redirectTo(Source: Loops[i]->getAfter(), Target: Loops[i + 1]->getPreheader(), DL);
7447 }
7448 redirectTo(Source: lastLoop->getPreheader(), Target: TCBlock, DL);
7449 redirectTo(Source: TCBlock, Target: fused->getPreheader(), DL);
7450 redirectTo(Source: fused->getAfter(), Target: lastLoop->getAfter(), DL);
7451
7452 // Build the fused body
7453 // Create new Blocks with conditions that jump to the original loop bodies
7454 SmallVector<BasicBlock *> condBBs;
7455 SmallVector<Value *> condValues;
7456 for (size_t i = 0; i < Loops.size(); ++i) {
7457 BasicBlock *condBlock = BasicBlock::Create(
7458 Context&: F->getContext(), Name: "omp.fused.inner.cond", Parent: F, InsertBefore: Loops[i]->getBody());
7459 Builder.SetInsertPoint(condBlock);
7460 Value *condValue =
7461 Builder.CreateICmpSLT(LHS: fused->getIndVar(), RHS: origTripCounts[i]);
7462 condBBs.push_back(Elt: condBlock);
7463 condValues.push_back(Elt: condValue);
7464 }
7465 // Join the condition blocks with the bodies of the original loops
7466 redirectTo(Source: fused->getBody(), Target: condBBs[0], DL);
7467 for (size_t i = 0; i < Loops.size() - 1; ++i) {
7468 Builder.SetInsertPoint(condBBs[i]);
7469 Builder.CreateCondBr(Cond: condValues[i], True: Loops[i]->getBody(), False: condBBs[i + 1]);
7470 redirectAllPredecessorsTo(OldTarget: Loops[i]->getLatch(), NewTarget: condBBs[i + 1], DL);
7471 // Replace the IV with the fused IV
7472 Loops[i]->getIndVar()->replaceAllUsesWith(V: fused->getIndVar());
7473 }
7474 // Last body jumps to the created end body block
7475 Builder.SetInsertPoint(condBBs.back());
7476 Builder.CreateCondBr(Cond: condValues.back(), True: lastLoop->getBody(),
7477 False: fused->getLatch());
7478 redirectAllPredecessorsTo(OldTarget: lastLoop->getLatch(), NewTarget: fused->getLatch(), DL);
7479 // Replace the IV with the fused IV
7480 lastLoop->getIndVar()->replaceAllUsesWith(V: fused->getIndVar());
7481
7482 // The loop latch must have only one predecessor. Currently it is branched to
7483 // from both the last condition block and the last loop body
7484 fused->getLatch()->splitBasicBlockBefore(I: fused->getLatch()->begin(),
7485 BBName: "omp.fused.pre_latch");
7486
7487 // Remove unused parts
7488 removeUnusedBlocksFromParent(BBs: oldControlBBs);
7489
7490 // Invalidate old CLIs
7491 for (CanonicalLoopInfo *L : Loops)
7492 L->invalidate();
7493
7494#ifndef NDEBUG
7495 fused->assertOK();
7496#endif
7497 return fused;
7498}
7499
7500void OpenMPIRBuilder::unrollLoopFull(DebugLoc, CanonicalLoopInfo *Loop) {
7501 LLVMContext &Ctx = Builder.getContext();
7502 addLoopMetadata(
7503 Loop, Properties: {MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.enable")),
7504 MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.full"))});
7505}
7506
7507void OpenMPIRBuilder::unrollLoopHeuristic(DebugLoc, CanonicalLoopInfo *Loop) {
7508 LLVMContext &Ctx = Builder.getContext();
7509 addLoopMetadata(
7510 Loop, Properties: {
7511 MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.enable")),
7512 });
7513}
7514
7515void OpenMPIRBuilder::createIfVersion(CanonicalLoopInfo *CanonicalLoop,
7516 Value *IfCond, ValueToValueMapTy &VMap,
7517 LoopAnalysis &LIA, LoopInfo &LI, Loop *L,
7518 const Twine &NamePrefix) {
7519 Function *F = CanonicalLoop->getFunction();
7520
7521 // We can't do
7522 // if (cond) {
7523 // simd_loop;
7524 // } else {
7525 // non_simd_loop;
7526 // }
7527 // because then the CanonicalLoopInfo would only point to one of the loops:
7528 // leading to other constructs operating on the same loop to malfunction.
7529 // Instead generate
7530 // while (...) {
7531 // if (cond) {
7532 // simd_body;
7533 // } else {
7534 // not_simd_body;
7535 // }
7536 // }
7537 // At least for simple loops, LLVM seems able to hoist the if out of the loop
7538 // body at -O3
7539
7540 // Define where if branch should be inserted
7541 auto SplitBeforeIt = CanonicalLoop->getBody()->getFirstNonPHIIt();
7542
7543 // Create additional blocks for the if statement
7544 BasicBlock *Cond = SplitBeforeIt->getParent();
7545 llvm::LLVMContext &C = Cond->getContext();
7546 llvm::BasicBlock *ThenBlock = llvm::BasicBlock::Create(
7547 Context&: C, Name: NamePrefix + ".if.then", Parent: Cond->getParent(), InsertBefore: Cond->getNextNode());
7548 llvm::BasicBlock *ElseBlock = llvm::BasicBlock::Create(
7549 Context&: C, Name: NamePrefix + ".if.else", Parent: Cond->getParent(), InsertBefore: CanonicalLoop->getExit());
7550
7551 // Create if condition branch.
7552 Builder.SetInsertPoint(SplitBeforeIt);
7553 Instruction *BrInstr =
7554 Builder.CreateCondBr(Cond: IfCond, True: ThenBlock, /*ifFalse*/ False: ElseBlock);
7555 InsertPointTy IP{BrInstr->getParent(), ++BrInstr->getIterator()};
7556 // Then block contains branch to omp loop body which needs to be vectorized
7557 spliceBB(IP, New: ThenBlock, CreateBranch: false, DL: Builder.getCurrentDebugLocation());
7558 ThenBlock->replaceSuccessorsPhiUsesWith(Old: Cond, New: ThenBlock);
7559
7560 Builder.SetInsertPoint(ElseBlock);
7561
7562 // Clone loop for the else branch
7563 SmallVector<BasicBlock *, 8> NewBlocks;
7564
7565 SmallVector<BasicBlock *, 8> ExistingBlocks;
7566 ExistingBlocks.reserve(N: L->getNumBlocks() + 1);
7567 ExistingBlocks.push_back(Elt: ThenBlock);
7568 ExistingBlocks.append(in_start: L->block_begin(), in_end: L->block_end());
7569 // Cond is the block that has the if clause condition
7570 // LoopCond is omp_loop.cond
7571 // LoopHeader is omp_loop.header
7572 BasicBlock *LoopCond = Cond->getUniquePredecessor();
7573 BasicBlock *LoopHeader = LoopCond->getUniquePredecessor();
7574 assert(LoopCond && LoopHeader && "Invalid loop structure");
7575 for (BasicBlock *Block : ExistingBlocks) {
7576 if (Block == L->getLoopPreheader() || Block == L->getLoopLatch() ||
7577 Block == LoopHeader || Block == LoopCond || Block == Cond) {
7578 continue;
7579 }
7580 BasicBlock *NewBB = CloneBasicBlock(BB: Block, VMap, NameSuffix: "", F);
7581
7582 // fix name not to be omp.if.then
7583 if (Block == ThenBlock)
7584 NewBB->setName(NamePrefix + ".if.else");
7585
7586 NewBB->moveBefore(MovePos: CanonicalLoop->getExit());
7587 VMap[Block] = NewBB;
7588 NewBlocks.push_back(Elt: NewBB);
7589 }
7590 remapInstructionsInBlocks(Blocks: NewBlocks, VMap);
7591 Builder.CreateBr(Dest: NewBlocks.front());
7592
7593 // The loop latch must have only one predecessor. Currently it is branched to
7594 // from both the 'then' and 'else' branches.
7595 L->getLoopLatch()->splitBasicBlockBefore(I: L->getLoopLatch()->begin(),
7596 BBName: NamePrefix + ".pre_latch");
7597
7598 // Ensure that the then block is added to the loop so we add the attributes in
7599 // the next step
7600 L->addBasicBlockToLoop(NewBB: ThenBlock, LI);
7601}
7602
7603unsigned
7604OpenMPIRBuilder::getOpenMPDefaultSimdAlign(const Triple &TargetTriple,
7605 const StringMap<bool> &Features) {
7606 if (TargetTriple.isX86()) {
7607 if (Features.lookup(Key: "avx512f"))
7608 return 512;
7609 else if (Features.lookup(Key: "avx"))
7610 return 256;
7611 return 128;
7612 }
7613 if (TargetTriple.isPPC())
7614 return 128;
7615 if (TargetTriple.isWasm())
7616 return 128;
7617 if (TargetTriple.isSystemZ())
7618 return 64;
7619 return 0;
7620}
7621
7622void OpenMPIRBuilder::applySimd(CanonicalLoopInfo *CanonicalLoop,
7623 MapVector<Value *, Value *> AlignedVars,
7624 Value *IfCond, OrderKind Order,
7625 ConstantInt *Simdlen, ConstantInt *Safelen) {
7626 LLVMContext &Ctx = Builder.getContext();
7627
7628 Function *F = CanonicalLoop->getFunction();
7629
7630 // Blocks must have terminators.
7631 // FIXME: Don't run analyses on incomplete/invalid IR.
7632 SmallVector<Instruction *> UIs;
7633 for (BasicBlock &BB : *F)
7634 if (!BB.hasTerminator())
7635 UIs.push_back(Elt: new UnreachableInst(F->getContext(), &BB));
7636
7637 // TODO: We should not rely on pass manager. Currently we use pass manager
7638 // only for getting llvm::Loop which corresponds to given CanonicalLoopInfo
7639 // object. We should have a method which returns all blocks between
7640 // CanonicalLoopInfo::getHeader() and CanonicalLoopInfo::getAfter()
7641 FunctionAnalysisManager FAM;
7642 FAM.registerPass(PassBuilder: []() { return DominatorTreeAnalysis(); });
7643 FAM.registerPass(PassBuilder: []() { return LoopAnalysis(); });
7644 FAM.registerPass(PassBuilder: []() { return PassInstrumentationAnalysis(); });
7645
7646 LoopAnalysis LIA;
7647 LoopInfo &&LI = LIA.run(F&: *F, AM&: FAM);
7648
7649 for (Instruction *I : UIs)
7650 I->eraseFromParent();
7651
7652 Loop *L = LI.getLoopFor(BB: CanonicalLoop->getHeader());
7653 if (AlignedVars.size()) {
7654 InsertPointTy IP = Builder.saveIP();
7655 for (auto &AlignedItem : AlignedVars) {
7656 Value *AlignedPtr = AlignedItem.first;
7657 Value *Alignment = AlignedItem.second;
7658 Instruction *loadInst = dyn_cast<Instruction>(Val: AlignedPtr);
7659 Builder.SetInsertPoint(loadInst->getNextNode());
7660 Builder.CreateAlignmentAssumption(DL: F->getDataLayout(), PtrValue: AlignedPtr,
7661 Alignment);
7662 }
7663 Builder.restoreIP(IP);
7664 }
7665
7666 if (IfCond) {
7667 ValueToValueMapTy VMap;
7668 createIfVersion(CanonicalLoop, IfCond, VMap, LIA, LI, L, NamePrefix: "simd");
7669 }
7670
7671 SmallPtrSet<BasicBlock *, 8> Reachable;
7672
7673 // Get the basic blocks from the loop in which memref instructions
7674 // can be found.
7675 // TODO: Generalize getting all blocks inside a CanonicalizeLoopInfo,
7676 // preferably without running any passes.
7677 for (BasicBlock *Block : L->getBlocks()) {
7678 if (Block == CanonicalLoop->getCond() ||
7679 Block == CanonicalLoop->getHeader())
7680 continue;
7681 Reachable.insert(Ptr: Block);
7682 }
7683
7684 SmallVector<Metadata *> LoopMDList;
7685
7686 // In presence of finite 'safelen', it may be unsafe to mark all
7687 // the memory instructions parallel, because loop-carried
7688 // dependences of 'safelen' iterations are possible.
7689 // If clause order(concurrent) is specified then the memory instructions
7690 // are marked parallel even if 'safelen' is finite.
7691 if ((Safelen == nullptr) || (Order == OrderKind::OMP_ORDER_concurrent))
7692 applyParallelAccessesMetadata(CLI: CanonicalLoop, Ctx, Loop: L, LoopInfo&: LI, LoopMDList);
7693
7694 // FIXME: the IF clause shares a loop backedge for the SIMD and non-SIMD
7695 // versions so we can't add the loop attributes in that case.
7696 if (IfCond) {
7697 // we can still add llvm.loop.parallel_access
7698 addLoopMetadata(Loop: CanonicalLoop, Properties: LoopMDList);
7699 return;
7700 }
7701
7702 // Use the above access group metadata to create loop level
7703 // metadata, which should be distinct for each loop.
7704 LoopMDList.push_back(
7705 Elt: MDNode::get(Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.vectorize.enable")}));
7706
7707 if (Simdlen || Safelen) {
7708 // If both simdlen and safelen clauses are specified, the value of the
7709 // simdlen parameter must be less than or equal to the value of the safelen
7710 // parameter. Therefore, use safelen only in the absence of simdlen.
7711 ConstantInt *VectorizeWidth = Simdlen == nullptr ? Safelen : Simdlen;
7712 LoopMDList.push_back(
7713 Elt: MDNode::get(Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.vectorize.width"),
7714 ConstantAsMetadata::get(C: VectorizeWidth)}));
7715 }
7716
7717 addLoopMetadata(Loop: CanonicalLoop, Properties: LoopMDList);
7718}
7719
7720/// Create the TargetMachine object to query the backend for optimization
7721/// preferences.
7722///
7723/// Ideally, this would be passed from the front-end to the OpenMPBuilder, but
7724/// e.g. Clang does not pass it to its CodeGen layer and creates it only when
7725/// needed for the LLVM pass pipline. We use some default options to avoid
7726/// having to pass too many settings from the frontend that probably do not
7727/// matter.
7728///
7729/// Currently, TargetMachine is only used sometimes by the unrollLoopPartial
7730/// method. If we are going to use TargetMachine for more purposes, especially
7731/// those that are sensitive to TargetOptions, RelocModel and CodeModel, it
7732/// might become be worth requiring front-ends to pass on their TargetMachine,
7733/// or at least cache it between methods. Note that while fontends such as Clang
7734/// have just a single main TargetMachine per translation unit, "target-cpu" and
7735/// "target-features" that determine the TargetMachine are per-function and can
7736/// be overrided using __attribute__((target("OPTIONS"))).
7737static std::unique_ptr<TargetMachine>
7738createTargetMachine(Function *F, CodeGenOptLevel OptLevel) {
7739 Module *M = F->getParent();
7740
7741 StringRef CPU = F->getFnAttribute(Kind: "target-cpu").getValueAsString();
7742 StringRef Features = F->getFnAttribute(Kind: "target-features").getValueAsString();
7743 const llvm::Triple &Triple = M->getTargetTriple();
7744
7745 std::string Error;
7746 const llvm::Target *TheTarget = TargetRegistry::lookupTarget(TheTriple: Triple, Error);
7747 if (!TheTarget)
7748 return {};
7749
7750 llvm::TargetOptions Options;
7751 return std::unique_ptr<TargetMachine>(TheTarget->createTargetMachine(
7752 TT: Triple, CPU, Features, Options, /*RelocModel=*/RM: std::nullopt,
7753 /*CodeModel=*/CM: std::nullopt, OL: OptLevel));
7754}
7755
7756/// Heuristically determine the best-performant unroll factor for \p CLI. This
7757/// depends on the target processor. We are re-using the same heuristics as the
7758/// LoopUnrollPass.
7759static int32_t computeHeuristicUnrollFactor(CanonicalLoopInfo *CLI) {
7760 Function *F = CLI->getFunction();
7761
7762 // Assume the user requests the most aggressive unrolling, even if the rest of
7763 // the code is optimized using a lower setting.
7764 CodeGenOptLevel OptLevel = CodeGenOptLevel::Aggressive;
7765 std::unique_ptr<TargetMachine> TM = createTargetMachine(F, OptLevel);
7766
7767 // Blocks must have terminators.
7768 // FIXME: Don't run analyses on incomplete/invalid IR.
7769 SmallVector<Instruction *> UIs;
7770 for (BasicBlock &BB : *F)
7771 if (!BB.hasTerminator())
7772 UIs.push_back(Elt: new UnreachableInst(F->getContext(), &BB));
7773
7774 FunctionAnalysisManager FAM;
7775 FAM.registerPass(PassBuilder: []() { return TargetLibraryAnalysis(); });
7776 FAM.registerPass(PassBuilder: []() { return AssumptionAnalysis(); });
7777 FAM.registerPass(PassBuilder: []() { return DominatorTreeAnalysis(); });
7778 FAM.registerPass(PassBuilder: []() { return LoopAnalysis(); });
7779 FAM.registerPass(PassBuilder: []() { return ScalarEvolutionAnalysis(); });
7780 FAM.registerPass(PassBuilder: []() { return PassInstrumentationAnalysis(); });
7781 TargetIRAnalysis TIRA;
7782 if (TM)
7783 TIRA = TargetIRAnalysis(
7784 [&](const Function &F) { return TM->getTargetTransformInfo(F); });
7785 FAM.registerPass(PassBuilder: [&]() { return TIRA; });
7786
7787 TargetIRAnalysis::Result &&TTI = TIRA.run(F: *F, FAM);
7788 ScalarEvolutionAnalysis SEA;
7789 ScalarEvolution &&SE = SEA.run(F&: *F, AM&: FAM);
7790 DominatorTreeAnalysis DTA;
7791 DominatorTree &&DT = DTA.run(F&: *F, FAM);
7792 LoopAnalysis LIA;
7793 LoopInfo &&LI = LIA.run(F&: *F, AM&: FAM);
7794 AssumptionAnalysis ACT;
7795 AssumptionCache &&AC = ACT.run(F&: *F, FAM);
7796 OptimizationRemarkEmitter ORE{F};
7797
7798 for (Instruction *I : UIs)
7799 I->eraseFromParent();
7800
7801 Loop *L = LI.getLoopFor(BB: CLI->getHeader());
7802 assert(L && "Expecting CanonicalLoopInfo to be recognized as a loop");
7803
7804 TargetTransformInfo::UnrollingPreferences UP = gatherUnrollingPreferences(
7805 L, SE, TTI,
7806 /*BlockFrequencyInfo=*/BFI: nullptr,
7807 /*ProfileSummaryInfo=*/PSI: nullptr, ORE, OptLevel: static_cast<int>(OptLevel),
7808 /*UserThreshold=*/std::nullopt,
7809 /*UserAllowPartial=*/true,
7810 /*UserAllowRuntime=*/UserRuntime: true,
7811 /*UserUpperBound=*/std::nullopt,
7812 /*UserFullUnrollMaxCount=*/std::nullopt);
7813
7814 UP.Force = true;
7815
7816 // Account for additional optimizations taking place before the LoopUnrollPass
7817 // would unroll the loop.
7818 UP.Threshold *= UnrollThresholdFactor;
7819 UP.PartialThreshold *= UnrollThresholdFactor;
7820
7821 // Use normal unroll factors even if the rest of the code is optimized for
7822 // size.
7823 UP.OptSizeThreshold = UP.Threshold;
7824 UP.PartialOptSizeThreshold = UP.PartialThreshold;
7825
7826 LLVM_DEBUG(dbgs() << "Unroll heuristic thresholds:\n"
7827 << " Threshold=" << UP.Threshold << "\n"
7828 << " PartialThreshold=" << UP.PartialThreshold << "\n"
7829 << " OptSizeThreshold=" << UP.OptSizeThreshold << "\n"
7830 << " PartialOptSizeThreshold="
7831 << UP.PartialOptSizeThreshold << "\n");
7832
7833 // Disable peeling.
7834 TargetTransformInfo::PeelingPreferences PP =
7835 gatherPeelingPreferences(L, SE, TTI,
7836 /*UserAllowPeeling=*/false,
7837 /*UserAllowProfileBasedPeeling=*/false,
7838 /*UnrollingSpecficValues=*/false);
7839
7840 SmallPtrSet<const Value *, 32> EphValues;
7841 CodeMetrics::collectEphemeralValues(L, AC: &AC, EphValues);
7842
7843 // Assume that reads and writes to stack variables can be eliminated by
7844 // Mem2Reg, SROA or LICM. That is, don't count them towards the loop body's
7845 // size.
7846 for (BasicBlock *BB : L->blocks()) {
7847 for (Instruction &I : *BB) {
7848 Value *Ptr;
7849 if (auto *Load = dyn_cast<LoadInst>(Val: &I)) {
7850 Ptr = Load->getPointerOperand();
7851 } else if (auto *Store = dyn_cast<StoreInst>(Val: &I)) {
7852 Ptr = Store->getPointerOperand();
7853 } else
7854 continue;
7855
7856 Ptr = Ptr->stripPointerCasts();
7857
7858 if (auto *Alloca = dyn_cast<AllocaInst>(Val: Ptr)) {
7859 if (Alloca->getParent() == &F->getEntryBlock())
7860 EphValues.insert(Ptr: &I);
7861 }
7862 }
7863 }
7864
7865 UnrollCostEstimator UCE(L, TTI, EphValues, UP.BEInsns);
7866
7867 // Loop is not unrollable if the loop contains certain instructions.
7868 if (!UCE.canUnroll()) {
7869 LLVM_DEBUG(dbgs() << "Loop not considered unrollable\n");
7870 return 1;
7871 }
7872
7873 LLVM_DEBUG(dbgs() << "Estimated loop size is " << UCE.getRolledLoopSize()
7874 << "\n");
7875
7876 // TODO: Determine trip count of \p CLI if constant, computeUnrollCount might
7877 // be able to use it.
7878 int TripCount = 0;
7879 int MaxTripCount = 0;
7880 bool MaxOrZero = false;
7881 unsigned TripMultiple = 0;
7882
7883 unsigned Factor =
7884 computeUnrollCount(L, TTI, DT, LI: &LI, AC: &AC, SE, EphValues, ORE: &ORE, TripCount,
7885 MaxTripCount, MaxOrZero, TripMultiple, UCE, UP, PP);
7886 LLVM_DEBUG(dbgs() << "Suggesting unroll factor of " << Factor << "\n");
7887
7888 // This function returns 1 to signal to not unroll a loop.
7889 if (Factor == 0)
7890 return 1;
7891 return Factor;
7892}
7893
7894void OpenMPIRBuilder::unrollLoopPartial(DebugLoc DL, CanonicalLoopInfo *Loop,
7895 int32_t Factor,
7896 CanonicalLoopInfo **UnrolledCLI) {
7897 assert(Factor >= 0 && "Unroll factor must not be negative");
7898
7899 Function *F = Loop->getFunction();
7900 LLVMContext &Ctx = F->getContext();
7901
7902 // If the unrolled loop is not used for another loop-associated directive, it
7903 // is sufficient to add metadata for the LoopUnrollPass.
7904 if (!UnrolledCLI) {
7905 SmallVector<Metadata *, 2> LoopMetadata;
7906 LoopMetadata.push_back(
7907 Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.enable")));
7908
7909 if (Factor >= 1) {
7910 ConstantAsMetadata *FactorConst = ConstantAsMetadata::get(
7911 C: ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: APInt(32, Factor)));
7912 LoopMetadata.push_back(Elt: MDNode::get(
7913 Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.count"), FactorConst}));
7914 }
7915
7916 addLoopMetadata(Loop, Properties: LoopMetadata);
7917 return;
7918 }
7919
7920 // Heuristically determine the unroll factor.
7921 if (Factor == 0)
7922 Factor = computeHeuristicUnrollFactor(CLI: Loop);
7923
7924 // No change required with unroll factor 1.
7925 if (Factor == 1) {
7926 *UnrolledCLI = Loop;
7927 return;
7928 }
7929
7930 assert(Factor >= 2 &&
7931 "unrolling only makes sense with a factor of 2 or larger");
7932
7933 Type *IndVarTy = Loop->getIndVarType();
7934
7935 // Apply partial unrolling by tiling the loop by the unroll-factor, then fully
7936 // unroll the inner loop.
7937 Value *FactorVal =
7938 ConstantInt::get(Ty: IndVarTy, V: APInt(IndVarTy->getIntegerBitWidth(), Factor,
7939 /*isSigned=*/false));
7940 std::vector<CanonicalLoopInfo *> LoopNest =
7941 tileLoops(DL, Loops: {Loop}, TileSizes: {FactorVal});
7942 assert(LoopNest.size() == 2 && "Expect 2 loops after tiling");
7943 *UnrolledCLI = LoopNest[0];
7944 CanonicalLoopInfo *InnerLoop = LoopNest[1];
7945
7946 // LoopUnrollPass can only fully unroll loops with constant trip count.
7947 // Unroll by the unroll factor with a fallback epilog for the remainder
7948 // iterations if necessary.
7949 ConstantAsMetadata *FactorConst = ConstantAsMetadata::get(
7950 C: ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: APInt(32, Factor)));
7951 addLoopMetadata(
7952 Loop: InnerLoop,
7953 Properties: {MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.enable")),
7954 MDNode::get(
7955 Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.count"), FactorConst})});
7956
7957#ifndef NDEBUG
7958 (*UnrolledCLI)->assertOK();
7959#endif
7960}
7961
7962OpenMPIRBuilder::InsertPointTy
7963OpenMPIRBuilder::createCopyPrivate(const LocationDescription &Loc,
7964 llvm::Value *BufSize, llvm::Value *CpyBuf,
7965 llvm::Value *CpyFn, llvm::Value *DidIt) {
7966 if (!updateToLocation(Loc))
7967 return Loc.IP;
7968
7969 uint32_t SrcLocStrSize;
7970 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
7971 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
7972 Value *ThreadId = getOrCreateThreadID(Ident);
7973
7974 llvm::Value *DidItLD = Builder.CreateLoad(Ty: Builder.getInt32Ty(), Ptr: DidIt);
7975
7976 Value *Args[] = {Ident, ThreadId, BufSize, CpyBuf, CpyFn, DidItLD};
7977
7978 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_copyprivate);
7979 createRuntimeFunctionCall(Callee: Fn, Args);
7980
7981 return Builder.saveIP();
7982}
7983
7984OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createSingle(
7985 const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,
7986 FinalizeCallbackTy FiniCB, bool IsNowait, ArrayRef<llvm::Value *> CPVars,
7987 ArrayRef<llvm::Function *> CPFuncs) {
7988
7989 if (!updateToLocation(Loc))
7990 return Loc.IP;
7991
7992 // If needed allocate and initialize `DidIt` with 0.
7993 // DidIt: flag variable: 1=single thread; 0=not single thread.
7994 llvm::Value *DidIt = nullptr;
7995 if (!CPVars.empty()) {
7996 DidIt = Builder.CreateAlloca(Ty: llvm::Type::getInt32Ty(C&: Builder.getContext()));
7997 Builder.CreateStore(Val: Builder.getInt32(C: 0), Ptr: DidIt);
7998 }
7999
8000 Directive OMPD = Directive::OMPD_single;
8001 uint32_t SrcLocStrSize;
8002 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8003 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8004 Value *ThreadId = getOrCreateThreadID(Ident);
8005 Value *Args[] = {Ident, ThreadId};
8006
8007 Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_single);
8008 Instruction *EntryCall = createRuntimeFunctionCall(Callee: EntryRTLFn, Args);
8009
8010 Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_end_single);
8011 Instruction *ExitCall = createRuntimeFunctionCall(Callee: ExitRTLFn, Args);
8012
8013 auto FiniCBWrapper = [&](InsertPointTy IP) -> Error {
8014 if (Error Err = FiniCB(IP))
8015 return Err;
8016
8017 // The thread that executes the single region must set `DidIt` to 1.
8018 // This is used by __kmpc_copyprivate, to know if the caller is the
8019 // single thread or not.
8020 if (DidIt)
8021 Builder.CreateStore(Val: Builder.getInt32(C: 1), Ptr: DidIt);
8022
8023 return Error::success();
8024 };
8025
8026 // generates the following:
8027 // if (__kmpc_single()) {
8028 // .... single region ...
8029 // __kmpc_end_single
8030 // }
8031 // __kmpc_copyprivate
8032 // __kmpc_barrier
8033
8034 InsertPointOrErrorTy AfterIP =
8035 EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB: FiniCBWrapper,
8036 /*Conditional*/ true,
8037 /*hasFinalize*/ HasFinalize: true);
8038 if (!AfterIP)
8039 return AfterIP.takeError();
8040
8041 if (DidIt) {
8042 for (size_t I = 0, E = CPVars.size(); I < E; ++I)
8043 // NOTE BufSize is currently unused, so just pass 0.
8044 createCopyPrivate(Loc: LocationDescription(Builder.saveIP(), Loc.DL),
8045 /*BufSize=*/ConstantInt::get(Ty: Int64, V: 0), CpyBuf: CPVars[I],
8046 CpyFn: CPFuncs[I], DidIt);
8047 // NOTE __kmpc_copyprivate already inserts a barrier
8048 } else if (!IsNowait) {
8049 InsertPointOrErrorTy AfterIP =
8050 createBarrier(Loc: LocationDescription(Builder.saveIP(), Loc.DL),
8051 Kind: omp::Directive::OMPD_unknown, /* ForceSimpleCall */ false,
8052 /* CheckCancelFlag */ false);
8053 if (!AfterIP)
8054 return AfterIP.takeError();
8055 }
8056 return Builder.saveIP();
8057}
8058
8059OpenMPIRBuilder::InsertPointOrErrorTy
8060OpenMPIRBuilder::createScope(const LocationDescription &Loc,
8061 BodyGenCallbackTy BodyGenCB,
8062 FinalizeCallbackTy FiniCB, bool IsNowait) {
8063
8064 if (!updateToLocation(Loc))
8065 return Loc.IP;
8066
8067 // All threads execute the scope body — no conditional entry.
8068 InsertPointOrErrorTy AfterIP = EmitOMPInlinedRegion(
8069 OMPD: Directive::OMPD_scope, /*EntryCall=*/nullptr, /*ExitCall=*/nullptr,
8070 BodyGenCB, FiniCB, /*Conditional=*/false, /*HasFinalize=*/true,
8071 /*IsCancellable=*/false);
8072 if (!AfterIP)
8073 return AfterIP.takeError();
8074
8075 Builder.restoreIP(IP: *AfterIP);
8076 if (!IsNowait) {
8077 AfterIP = createBarrier(Loc: LocationDescription(Builder.saveIP(), Loc.DL),
8078 Kind: omp::Directive::OMPD_unknown,
8079 /*ForceSimpleCall=*/false,
8080 /*CheckCancelFlag=*/false);
8081 if (!AfterIP)
8082 return AfterIP.takeError();
8083 }
8084 return Builder.saveIP();
8085}
8086
8087OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createCritical(
8088 const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,
8089 FinalizeCallbackTy FiniCB, StringRef CriticalName, Value *HintInst) {
8090
8091 if (!updateToLocation(Loc))
8092 return Loc.IP;
8093
8094 Directive OMPD = Directive::OMPD_critical;
8095 uint32_t SrcLocStrSize;
8096 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8097 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8098 Value *ThreadId = getOrCreateThreadID(Ident);
8099 Value *LockVar = getOMPCriticalRegionLock(CriticalName);
8100 Value *Args[] = {Ident, ThreadId, LockVar};
8101
8102 SmallVector<llvm::Value *, 4> EnterArgs(std::begin(arr&: Args), std::end(arr&: Args));
8103 Function *RTFn = nullptr;
8104 if (HintInst) {
8105 // Add Hint to entry Args and create call
8106 EnterArgs.push_back(Elt: HintInst);
8107 RTFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_critical_with_hint);
8108 } else {
8109 RTFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_critical);
8110 }
8111 Instruction *EntryCall = createRuntimeFunctionCall(Callee: RTFn, Args: EnterArgs);
8112
8113 Function *ExitRTLFn =
8114 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_end_critical);
8115 Instruction *ExitCall = createRuntimeFunctionCall(Callee: ExitRTLFn, Args);
8116
8117 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
8118 /*Conditional*/ false, /*hasFinalize*/ HasFinalize: true);
8119}
8120
8121OpenMPIRBuilder::InsertPointTy
8122OpenMPIRBuilder::createOrderedDepend(const LocationDescription &Loc,
8123 InsertPointTy AllocaIP, unsigned NumLoops,
8124 ArrayRef<llvm::Value *> StoreValues,
8125 const Twine &Name, bool IsDependSource) {
8126 assert(
8127 llvm::all_of(StoreValues,
8128 [](Value *SV) { return SV->getType()->isIntegerTy(64); }) &&
8129 "OpenMP runtime requires depend vec with i64 type");
8130
8131 if (!updateToLocation(Loc))
8132 return Loc.IP;
8133
8134 // Allocate space for vector and generate alloc instruction.
8135 auto *ArrI64Ty = ArrayType::get(ElementType: Int64, NumElements: NumLoops);
8136 Builder.restoreIP(IP: AllocaIP);
8137 AllocaInst *ArgsBase = Builder.CreateAlloca(Ty: ArrI64Ty, ArraySize: nullptr, Name);
8138 ArgsBase->setAlignment(Align(8));
8139 updateToLocation(Loc);
8140
8141 // Store the index value with offset in depend vector.
8142 for (unsigned I = 0; I < NumLoops; ++I) {
8143 Value *DependAddrGEPIter = Builder.CreateInBoundsGEP(
8144 Ty: ArrI64Ty, Ptr: ArgsBase, IdxList: {Builder.getInt64(C: 0), Builder.getInt64(C: I)});
8145 StoreInst *STInst = Builder.CreateStore(Val: StoreValues[I], Ptr: DependAddrGEPIter);
8146 STInst->setAlignment(Align(8));
8147 }
8148
8149 Value *DependBaseAddrGEP = Builder.CreateInBoundsGEP(
8150 Ty: ArrI64Ty, Ptr: ArgsBase, IdxList: {Builder.getInt64(C: 0), Builder.getInt64(C: 0)});
8151
8152 uint32_t SrcLocStrSize;
8153 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8154 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8155 Value *ThreadId = getOrCreateThreadID(Ident);
8156 Value *Args[] = {Ident, ThreadId, DependBaseAddrGEP};
8157
8158 Function *RTLFn = nullptr;
8159 if (IsDependSource)
8160 RTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_doacross_post);
8161 else
8162 RTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_doacross_wait);
8163 createRuntimeFunctionCall(Callee: RTLFn, Args);
8164
8165 return Builder.saveIP();
8166}
8167
8168OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createOrderedThreadsSimd(
8169 const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,
8170 FinalizeCallbackTy FiniCB, bool IsThreads) {
8171 if (!updateToLocation(Loc))
8172 return Loc.IP;
8173
8174 Directive OMPD = Directive::OMPD_ordered_blockassoc;
8175 Instruction *EntryCall = nullptr;
8176 Instruction *ExitCall = nullptr;
8177
8178 if (IsThreads) {
8179 uint32_t SrcLocStrSize;
8180 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8181 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8182 Value *ThreadId = getOrCreateThreadID(Ident);
8183 Value *Args[] = {Ident, ThreadId};
8184
8185 Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_ordered);
8186 EntryCall = createRuntimeFunctionCall(Callee: EntryRTLFn, Args);
8187
8188 Function *ExitRTLFn =
8189 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_end_ordered);
8190 ExitCall = createRuntimeFunctionCall(Callee: ExitRTLFn, Args);
8191 }
8192
8193 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
8194 /*Conditional*/ false, /*hasFinalize*/ HasFinalize: true);
8195}
8196
8197OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::EmitOMPInlinedRegion(
8198 Directive OMPD, Instruction *EntryCall, Instruction *ExitCall,
8199 BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB, bool Conditional,
8200 bool HasFinalize, bool IsCancellable) {
8201
8202 if (HasFinalize)
8203 FinalizationStack.push_back(Elt: {FiniCB, OMPD, IsCancellable});
8204
8205 // Create inlined region's entry and body blocks, in preparation
8206 // for conditional creation
8207 BasicBlock *EntryBB = Builder.GetInsertBlock();
8208 Instruction *SplitPos = EntryBB->getTerminatorOrNull();
8209 if (!isa_and_nonnull<UncondBrInst, CondBrInst>(Val: SplitPos))
8210 SplitPos = new UnreachableInst(Builder.getContext(), EntryBB);
8211 BasicBlock *ExitBB = EntryBB->splitBasicBlock(I: SplitPos, BBName: "omp_region.end");
8212 BasicBlock *FiniBB =
8213 EntryBB->splitBasicBlock(I: EntryBB->getTerminator(), BBName: "omp_region.finalize");
8214
8215 Builder.SetInsertPoint(EntryBB->getTerminator());
8216 emitCommonDirectiveEntry(OMPD, EntryCall, ExitBB, Conditional);
8217
8218 // generate body
8219 if (Error Err =
8220 BodyGenCB(/* AllocaIP */ InsertPointTy(),
8221 /* CodeGenIP */ Builder.saveIP(), /* DeallocBlocks */ {}))
8222 return Err;
8223
8224 // emit exit call and do any needed finalization.
8225 auto FinIP = InsertPointTy(FiniBB, FiniBB->getFirstInsertionPt());
8226 assert(FiniBB->getTerminator()->getNumSuccessors() == 1 &&
8227 FiniBB->getTerminator()->getSuccessor(0) == ExitBB &&
8228 "Unexpected control flow graph state!!");
8229 InsertPointOrErrorTy AfterIP =
8230 emitCommonDirectiveExit(OMPD, FinIP, ExitCall, HasFinalize);
8231 if (!AfterIP)
8232 return AfterIP.takeError();
8233
8234 // If we are skipping the region of a non conditional, remove the exit
8235 // block, and clear the builder's insertion point.
8236 assert(SplitPos->getParent() == ExitBB &&
8237 "Unexpected Insertion point location!");
8238 auto merged = MergeBlockIntoPredecessor(BB: ExitBB);
8239 BasicBlock *ExitPredBB = SplitPos->getParent();
8240 auto InsertBB = merged ? ExitPredBB : ExitBB;
8241 if (!isa_and_nonnull<UncondBrInst, CondBrInst>(Val: SplitPos))
8242 SplitPos->eraseFromParent();
8243 Builder.SetInsertPoint(InsertBB);
8244
8245 return Builder.saveIP();
8246}
8247
8248OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitCommonDirectiveEntry(
8249 Directive OMPD, Value *EntryCall, BasicBlock *ExitBB, bool Conditional) {
8250 // if nothing to do, Return current insertion point.
8251 if (!Conditional || !EntryCall)
8252 return Builder.saveIP();
8253
8254 BasicBlock *EntryBB = Builder.GetInsertBlock();
8255 Value *CallBool = Builder.CreateIsNotNull(Arg: EntryCall);
8256 auto *ThenBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp_region.body");
8257 auto *UI = new UnreachableInst(Builder.getContext(), ThenBB);
8258
8259 // Emit thenBB and set the Builder's insertion point there for
8260 // body generation next. Place the block after the current block.
8261 Function *CurFn = EntryBB->getParent();
8262 CurFn->insert(Position: std::next(x: EntryBB->getIterator()), BB: ThenBB);
8263
8264 // Move Entry branch to end of ThenBB, and replace with conditional
8265 // branch (If-stmt)
8266 Instruction *EntryBBTI = EntryBB->getTerminator();
8267 Builder.CreateCondBr(Cond: CallBool, True: ThenBB, False: ExitBB);
8268 EntryBBTI->removeFromParent();
8269 Builder.SetInsertPoint(UI);
8270 Builder.Insert(I: EntryBBTI);
8271 UI->eraseFromParent();
8272 Builder.SetInsertPoint(ThenBB->getTerminator());
8273
8274 // return an insertion point to ExitBB.
8275 return IRBuilder<>::InsertPoint(ExitBB, ExitBB->getFirstInsertionPt());
8276}
8277
8278OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitCommonDirectiveExit(
8279 omp::Directive OMPD, InsertPointTy FinIP, Instruction *ExitCall,
8280 bool HasFinalize) {
8281
8282 Builder.restoreIP(IP: FinIP);
8283
8284 // If there is finalization to do, emit it before the exit call
8285 if (HasFinalize) {
8286 assert(!FinalizationStack.empty() &&
8287 "Unexpected finalization stack state!");
8288
8289 FinalizationInfo Fi = FinalizationStack.pop_back_val();
8290 assert(Fi.DK == OMPD && "Unexpected Directive for Finalization call!");
8291
8292 if (Error Err = Fi.mergeFiniBB(Builder, OtherFiniBB: FinIP.getBlock()))
8293 return std::move(Err);
8294
8295 // Exit condition: insertion point is before the terminator of the new Fini
8296 // block
8297 Builder.SetInsertPoint(FinIP.getBlock()->getTerminator());
8298 }
8299
8300 if (!ExitCall)
8301 return Builder.saveIP();
8302
8303 // place the Exitcall as last instruction before Finalization block terminator
8304 ExitCall->removeFromParent();
8305 Builder.Insert(I: ExitCall);
8306
8307 return IRBuilder<>::InsertPoint(ExitCall->getParent(),
8308 ExitCall->getIterator());
8309}
8310
8311OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createCopyinClauseBlocks(
8312 InsertPointTy IP, Value *MasterAddr, Value *PrivateAddr,
8313 llvm::IntegerType *IntPtrTy, bool BranchtoEnd) {
8314 if (!IP.isSet())
8315 return IP;
8316
8317 IRBuilder<>::InsertPointGuard IPG(Builder);
8318
8319 // creates the following CFG structure
8320 // OMP_Entry : (MasterAddr != PrivateAddr)?
8321 // F T
8322 // | \
8323 // | copin.not.master
8324 // | /
8325 // v /
8326 // copyin.not.master.end
8327 // |
8328 // v
8329 // OMP.Entry.Next
8330
8331 BasicBlock *OMP_Entry = IP.getBlock();
8332 Function *CurFn = OMP_Entry->getParent();
8333 BasicBlock *CopyBegin =
8334 BasicBlock::Create(Context&: M.getContext(), Name: "copyin.not.master", Parent: CurFn);
8335 BasicBlock *CopyEnd = nullptr;
8336
8337 // If entry block is terminated, split to preserve the branch to following
8338 // basic block (i.e. OMP.Entry.Next), otherwise, leave everything as is.
8339 if (isa_and_nonnull<CondBrInst>(Val: OMP_Entry->getTerminatorOrNull())) {
8340 CopyEnd = OMP_Entry->splitBasicBlock(I: OMP_Entry->getTerminator(),
8341 BBName: "copyin.not.master.end");
8342 OMP_Entry->getTerminator()->eraseFromParent();
8343 } else {
8344 CopyEnd =
8345 BasicBlock::Create(Context&: M.getContext(), Name: "copyin.not.master.end", Parent: CurFn);
8346 }
8347
8348 Builder.SetInsertPoint(OMP_Entry);
8349 Value *MasterPtr = Builder.CreatePtrToInt(V: MasterAddr, DestTy: IntPtrTy);
8350 Value *PrivatePtr = Builder.CreatePtrToInt(V: PrivateAddr, DestTy: IntPtrTy);
8351 Value *cmp = Builder.CreateICmpNE(LHS: MasterPtr, RHS: PrivatePtr);
8352 Builder.CreateCondBr(Cond: cmp, True: CopyBegin, False: CopyEnd);
8353
8354 Builder.SetInsertPoint(CopyBegin);
8355 if (BranchtoEnd)
8356 Builder.SetInsertPoint(Builder.CreateBr(Dest: CopyEnd));
8357
8358 return Builder.saveIP();
8359}
8360
8361CallInst *OpenMPIRBuilder::createOMPAlloc(const LocationDescription &Loc,
8362 Value *Size, Value *Allocator,
8363 std::string Name) {
8364 IRBuilder<>::InsertPointGuard IPG(Builder);
8365 if (!updateToLocation(Loc))
8366 return nullptr;
8367
8368 uint32_t SrcLocStrSize;
8369 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8370 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8371 Value *ThreadId = getOrCreateThreadID(Ident);
8372 Value *Args[] = {ThreadId, Size, Allocator};
8373
8374 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_alloc);
8375
8376 return createRuntimeFunctionCall(Callee: Fn, Args, Name);
8377}
8378
8379CallInst *OpenMPIRBuilder::createOMPAlignedAlloc(const LocationDescription &Loc,
8380 Value *Align, Value *Size,
8381 Value *Allocator,
8382 std::string Name) {
8383 IRBuilder<>::InsertPointGuard IPG(Builder);
8384 if (!updateToLocation(Loc))
8385 return nullptr;
8386
8387 uint32_t SrcLocStrSize;
8388 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8389 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8390 Value *ThreadId = getOrCreateThreadID(Ident);
8391 Value *Args[] = {ThreadId, Align, Size, Allocator};
8392
8393 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_aligned_alloc);
8394
8395 return Builder.CreateCall(Callee: Fn, Args, Name);
8396}
8397
8398CallInst *OpenMPIRBuilder::createOMPFree(const LocationDescription &Loc,
8399 Value *Addr, Value *Allocator,
8400 std::string Name) {
8401 IRBuilder<>::InsertPointGuard IPG(Builder);
8402 if (!updateToLocation(Loc))
8403 return nullptr;
8404
8405 uint32_t SrcLocStrSize;
8406 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8407 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8408 Value *ThreadId = getOrCreateThreadID(Ident);
8409 Value *Args[] = {ThreadId, Addr, Allocator};
8410 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_free);
8411 return createRuntimeFunctionCall(Callee: Fn, Args, Name);
8412}
8413
8414CallInst *OpenMPIRBuilder::createOMPAllocShared(const LocationDescription &Loc,
8415 Value *Size,
8416 const Twine &Name) {
8417 IRBuilder<>::InsertPointGuard IPG(Builder);
8418 updateToLocation(Loc);
8419
8420 Value *Args[] = {Size};
8421 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_alloc_shared);
8422 CallInst *Call = Builder.CreateCall(Callee: Fn, Args, Name);
8423 Call->addRetAttr(Attr: Attribute::getWithAlignment(
8424 Context&: M.getContext(), Alignment: M.getDataLayout().getPrefTypeAlign(Ty: Int64)));
8425 return Call;
8426}
8427
8428CallInst *OpenMPIRBuilder::createOMPAllocShared(const LocationDescription &Loc,
8429 Type *VarType,
8430 const Twine &Name) {
8431 return createOMPAllocShared(
8432 Loc, Size: Builder.getInt64(C: M.getDataLayout().getTypeAllocSize(Ty: VarType)), Name);
8433}
8434
8435CallInst *OpenMPIRBuilder::createOMPFreeShared(const LocationDescription &Loc,
8436 Value *Addr, Value *Size,
8437 const Twine &Name) {
8438 IRBuilder<>::InsertPointGuard IPG(Builder);
8439 updateToLocation(Loc);
8440
8441 Value *Args[] = {Addr, Size};
8442 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_free_shared);
8443 return Builder.CreateCall(Callee: Fn, Args, Name);
8444}
8445
8446CallInst *OpenMPIRBuilder::createOMPFreeShared(const LocationDescription &Loc,
8447 Value *Addr, Type *VarType,
8448 const Twine &Name) {
8449 return createOMPFreeShared(
8450 Loc, Addr, Size: Builder.getInt64(C: M.getDataLayout().getTypeAllocSize(Ty: VarType)),
8451 Name);
8452}
8453
8454CallInst *OpenMPIRBuilder::createOMPInteropInit(
8455 const LocationDescription &Loc, Value *InteropVar,
8456 omp::OMPInteropType InteropType, Value *Device, Value *NumDependences,
8457 Value *DependenceAddress, bool HaveNowaitClause) {
8458 IRBuilder<>::InsertPointGuard IPG(Builder);
8459 updateToLocation(Loc);
8460
8461 uint32_t SrcLocStrSize;
8462 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8463 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8464 Value *ThreadId = getOrCreateThreadID(Ident);
8465 if (Device == nullptr)
8466 Device = Constant::getAllOnesValue(Ty: Int32);
8467 else if (Device->getType() != Int32)
8468 Device = Builder.CreateIntCast(V: Device, DestTy: Int32, /*isSigned=*/true);
8469 Constant *InteropTypeVal = ConstantInt::get(Ty: Int32, V: (int)InteropType);
8470 if (NumDependences == nullptr) {
8471 NumDependences = ConstantInt::get(Ty: Int32, V: 0);
8472 PointerType *PointerTypeVar = PointerType::getUnqual(C&: M.getContext());
8473 DependenceAddress = ConstantPointerNull::get(T: PointerTypeVar);
8474 }
8475 Value *HaveNowaitClauseVal = ConstantInt::get(Ty: Int32, V: HaveNowaitClause);
8476 Value *Args[] = {
8477 Ident, ThreadId, InteropVar, InteropTypeVal,
8478 Device, NumDependences, DependenceAddress, HaveNowaitClauseVal};
8479
8480 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___tgt_interop_init);
8481
8482 return createRuntimeFunctionCall(Callee: Fn, Args);
8483}
8484
8485CallInst *OpenMPIRBuilder::createOMPInteropDestroy(
8486 const LocationDescription &Loc, Value *InteropVar, Value *Device,
8487 Value *NumDependences, Value *DependenceAddress, bool HaveNowaitClause) {
8488 IRBuilder<>::InsertPointGuard IPG(Builder);
8489 updateToLocation(Loc);
8490
8491 uint32_t SrcLocStrSize;
8492 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8493 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8494 Value *ThreadId = getOrCreateThreadID(Ident);
8495 if (Device == nullptr)
8496 Device = Constant::getAllOnesValue(Ty: Int32);
8497 else if (Device->getType() != Int32)
8498 Device = Builder.CreateIntCast(V: Device, DestTy: Int32, /*isSigned=*/true);
8499 if (NumDependences == nullptr) {
8500 NumDependences = ConstantInt::get(Ty: Int32, V: 0);
8501 PointerType *PointerTypeVar = PointerType::getUnqual(C&: M.getContext());
8502 DependenceAddress = ConstantPointerNull::get(T: PointerTypeVar);
8503 }
8504 Value *HaveNowaitClauseVal = ConstantInt::get(Ty: Int32, V: HaveNowaitClause);
8505 Value *Args[] = {
8506 Ident, ThreadId, InteropVar, Device,
8507 NumDependences, DependenceAddress, HaveNowaitClauseVal};
8508
8509 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___tgt_interop_destroy);
8510
8511 return createRuntimeFunctionCall(Callee: Fn, Args);
8512}
8513
8514CallInst *OpenMPIRBuilder::createOMPInteropUse(const LocationDescription &Loc,
8515 Value *InteropVar, Value *Device,
8516 Value *NumDependences,
8517 Value *DependenceAddress,
8518 bool HaveNowaitClause) {
8519 IRBuilder<>::InsertPointGuard IPG(Builder);
8520 updateToLocation(Loc);
8521 uint32_t SrcLocStrSize;
8522 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8523 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8524 Value *ThreadId = getOrCreateThreadID(Ident);
8525 if (Device == nullptr)
8526 Device = Constant::getAllOnesValue(Ty: Int32);
8527 else if (Device->getType() != Int32)
8528 Device = Builder.CreateIntCast(V: Device, DestTy: Int32, /*isSigned=*/true);
8529 if (NumDependences == nullptr) {
8530 NumDependences = ConstantInt::get(Ty: Int32, V: 0);
8531 PointerType *PointerTypeVar = PointerType::getUnqual(C&: M.getContext());
8532 DependenceAddress = ConstantPointerNull::get(T: PointerTypeVar);
8533 }
8534 Value *HaveNowaitClauseVal = ConstantInt::get(Ty: Int32, V: HaveNowaitClause);
8535 Value *Args[] = {
8536 Ident, ThreadId, InteropVar, Device,
8537 NumDependences, DependenceAddress, HaveNowaitClauseVal};
8538
8539 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___tgt_interop_use);
8540
8541 return createRuntimeFunctionCall(Callee: Fn, Args);
8542}
8543
8544CallInst *OpenMPIRBuilder::createCachedThreadPrivate(
8545 const LocationDescription &Loc, llvm::Value *Pointer,
8546 llvm::ConstantInt *Size, const llvm::Twine &Name) {
8547 IRBuilder<>::InsertPointGuard IPG(Builder);
8548 updateToLocation(Loc);
8549
8550 uint32_t SrcLocStrSize;
8551 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8552 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8553 Value *ThreadId = getOrCreateThreadID(Ident);
8554 Constant *ThreadPrivateCache =
8555 getOrCreateInternalVariable(Ty: Int8PtrPtr, Name: Name.str());
8556 llvm::Value *Args[] = {Ident, ThreadId, Pointer, Size, ThreadPrivateCache};
8557
8558 Function *Fn =
8559 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_threadprivate_cached);
8560
8561 return createRuntimeFunctionCall(Callee: Fn, Args);
8562}
8563
8564OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTargetInit(
8565 const LocationDescription &Loc,
8566 const llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs &Attrs) {
8567 assert(!Attrs.MaxThreads.empty() && !Attrs.MaxTeams.empty() &&
8568 "expected num_threads and num_teams to be specified");
8569
8570 if (!updateToLocation(Loc))
8571 return Loc.IP;
8572
8573 uint32_t SrcLocStrSize;
8574 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8575 Constant *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8576 Constant *IsSPMDVal = ConstantInt::getSigned(Ty: Int8, V: Attrs.ExecFlags);
8577 Constant *UseGenericStateMachineVal = ConstantInt::getSigned(
8578 Ty: Int8, V: Attrs.ExecFlags != omp::OMP_TGT_EXEC_MODE_SPMD &&
8579 Attrs.ExecFlags != omp::OMP_TGT_EXEC_MODE_SPMD_NO_LOOP);
8580 Constant *MayUseNestedParallelismVal = ConstantInt::getSigned(Ty: Int8, V: true);
8581 Constant *DebugIndentionLevelVal = ConstantInt::getSigned(Ty: Int16, V: 0);
8582
8583 Function *DebugKernelWrapper = Builder.GetInsertBlock()->getParent();
8584 Function *Kernel = DebugKernelWrapper;
8585
8586 // We need to strip the debug prefix to get the correct kernel name.
8587 StringRef KernelName = Kernel->getName();
8588 const std::string DebugPrefix = "_debug__";
8589 if (KernelName.ends_with(Suffix: DebugPrefix)) {
8590 KernelName = KernelName.drop_back(N: DebugPrefix.length());
8591 Kernel = M.getFunction(Name: KernelName);
8592 assert(Kernel && "Expected the real kernel to exist");
8593 }
8594
8595 // Manifest the launch configuration in the metadata matching the kernel
8596 // environment.
8597 if (Attrs.MinTeams.front() > 1 || Attrs.MaxTeams.front() > 0)
8598 writeTeamsForKernel(T, Kernel&: *Kernel, LB: Attrs.MinTeams.front(),
8599 UB: Attrs.MaxTeams.front());
8600
8601 // If MaxThreads is not set and needs adjustment, select the maximum between
8602 // the default workgroup size and the MinThreads value.
8603 int32_t MaxThreadsVal = Attrs.MaxThreads.front();
8604 if (MaxThreadsVal < 0 && UseDefaultMaxThreads) {
8605 if (hasGridValue(T)) {
8606 MaxThreadsVal =
8607 std::max(a: int32_t(getGridValue(T, Kernel).GV_Default_WG_Size),
8608 b: Attrs.MinThreads.front());
8609 } else {
8610 MaxThreadsVal = Attrs.MinThreads.front();
8611 }
8612 }
8613
8614 // Generic mode runs the main thread on a warp of its own, past thread_limit.
8615 // Reserve the widest warp any target has. Not on SPIR-V, causes problems with
8616 // Level Zero.
8617 if (MaxThreadsVal > 0 && Attrs.ExecFlags == omp::OMP_TGT_EXEC_MODE_GENERIC &&
8618 hasGridValue(T) && !T.isSPIRV())
8619 MaxThreadsVal = int32_t(
8620 std::min<int64_t>(a: int64_t(MaxThreadsVal) + 64,
8621 b: int64_t(getGridValue(T, Kernel).GV_Max_WG_Size)));
8622
8623 if (MaxThreadsVal > 0)
8624 writeThreadBoundsForKernel(T, Kernel&: *Kernel, LB: Attrs.MinThreads.front(),
8625 UB: MaxThreadsVal);
8626
8627 Constant *MinThreads =
8628 ConstantInt::getSigned(Ty: Int32, V: Attrs.MinThreads.front());
8629 Constant *MaxThreads = ConstantInt::getSigned(Ty: Int32, V: MaxThreadsVal);
8630 Constant *MinTeams = ConstantInt::getSigned(Ty: Int32, V: Attrs.MinTeams.front());
8631 Constant *MaxTeams = ConstantInt::getSigned(Ty: Int32, V: Attrs.MaxTeams.front());
8632 Constant *ReductionDataSize =
8633 ConstantInt::getSigned(Ty: Int32, V: Attrs.ReductionDataSize);
8634
8635 Function *Fn = getOrCreateRuntimeFunctionPtr(
8636 FnID: omp::RuntimeFunction::OMPRTL___kmpc_target_init);
8637 const DataLayout &DL = Fn->getDataLayout();
8638
8639 Twine DynamicEnvironmentName = KernelName + "_dynamic_environment";
8640 Constant *DynamicEnvironmentInitializer =
8641 ConstantStruct::get(T: DynamicEnvironment, V: {DebugIndentionLevelVal});
8642 GlobalVariable *DynamicEnvironmentGV = new GlobalVariable(
8643 M, DynamicEnvironment, /*IsConstant=*/false, GlobalValue::WeakODRLinkage,
8644 DynamicEnvironmentInitializer, DynamicEnvironmentName,
8645 /*InsertBefore=*/nullptr, GlobalValue::NotThreadLocal,
8646 DL.getDefaultGlobalsAddressSpace());
8647 DynamicEnvironmentGV->setVisibility(GlobalValue::ProtectedVisibility);
8648
8649 Constant *DynamicEnvironment =
8650 DynamicEnvironmentGV->getType() == DynamicEnvironmentPtr
8651 ? DynamicEnvironmentGV
8652 : ConstantExpr::getAddrSpaceCast(C: DynamicEnvironmentGV,
8653 Ty: DynamicEnvironmentPtr);
8654
8655 Constant *ConfigurationEnvironmentInitializer = ConstantStruct::get(
8656 T: ConfigurationEnvironment, V: {
8657 UseGenericStateMachineVal,
8658 MayUseNestedParallelismVal,
8659 IsSPMDVal,
8660 MinThreads,
8661 MaxThreads,
8662 MinTeams,
8663 MaxTeams,
8664 ReductionDataSize,
8665 });
8666 Constant *KernelEnvironmentInitializer = ConstantStruct::get(
8667 T: KernelEnvironment, V: {
8668 ConfigurationEnvironmentInitializer,
8669 Ident,
8670 DynamicEnvironment,
8671 });
8672 std::string KernelEnvironmentName =
8673 (KernelName + "_kernel_environment").str();
8674 GlobalVariable *KernelEnvironmentGV = new GlobalVariable(
8675 M, KernelEnvironment, /*IsConstant=*/true, GlobalValue::WeakODRLinkage,
8676 KernelEnvironmentInitializer, KernelEnvironmentName,
8677 /*InsertBefore=*/nullptr, GlobalValue::NotThreadLocal,
8678 DL.getDefaultGlobalsAddressSpace());
8679 KernelEnvironmentGV->setVisibility(GlobalValue::ProtectedVisibility);
8680
8681 Constant *KernelEnvironment =
8682 KernelEnvironmentGV->getType() == KernelEnvironmentPtr
8683 ? KernelEnvironmentGV
8684 : ConstantExpr::getAddrSpaceCast(C: KernelEnvironmentGV,
8685 Ty: KernelEnvironmentPtr);
8686 Value *KernelLaunchEnvironment =
8687 DebugKernelWrapper->getArg(i: DebugKernelWrapper->arg_size() - 1);
8688 Type *KernelLaunchEnvParamTy = Fn->getFunctionType()->getParamType(i: 1);
8689 KernelLaunchEnvironment =
8690 KernelLaunchEnvironment->getType() == KernelLaunchEnvParamTy
8691 ? KernelLaunchEnvironment
8692 : Builder.CreateAddrSpaceCast(V: KernelLaunchEnvironment,
8693 DestTy: KernelLaunchEnvParamTy);
8694 CallInst *ThreadKind = createRuntimeFunctionCall(
8695 Callee: Fn, Args: {KernelEnvironment, KernelLaunchEnvironment});
8696
8697 Value *ExecUserCode = Builder.CreateICmpEQ(
8698 LHS: ThreadKind, RHS: Constant::getAllOnesValue(Ty: ThreadKind->getType()),
8699 Name: "exec_user_code");
8700
8701 // ThreadKind = __kmpc_target_init(...)
8702 // if (ThreadKind == -1)
8703 // user_code
8704 // else
8705 // return;
8706
8707 auto *UI = Builder.CreateUnreachable();
8708 BasicBlock *CheckBB = UI->getParent();
8709 BasicBlock *UserCodeEntryBB = CheckBB->splitBasicBlock(I: UI, BBName: "user_code.entry");
8710
8711 BasicBlock *WorkerExitBB = BasicBlock::Create(
8712 Context&: CheckBB->getContext(), Name: "worker.exit", Parent: CheckBB->getParent());
8713 Builder.SetInsertPoint(WorkerExitBB);
8714 Builder.CreateRetVoid();
8715
8716 auto *CheckBBTI = CheckBB->getTerminator();
8717 Builder.SetInsertPoint(CheckBBTI);
8718 Builder.CreateCondBr(Cond: ExecUserCode, True: UI->getParent(), False: WorkerExitBB);
8719
8720 CheckBBTI->eraseFromParent();
8721 UI->eraseFromParent();
8722
8723 // Continue in the "user_code" block, see diagram above and in
8724 // openmp/libomptarget/deviceRTLs/common/include/target.h .
8725 return InsertPointTy(UserCodeEntryBB, UserCodeEntryBB->getFirstInsertionPt());
8726}
8727
8728void OpenMPIRBuilder::createTargetDeinit(const LocationDescription &Loc,
8729 int32_t TeamsReductionDataSize) {
8730 if (!updateToLocation(Loc))
8731 return;
8732
8733 Function *Fn = getOrCreateRuntimeFunctionPtr(
8734 FnID: omp::RuntimeFunction::OMPRTL___kmpc_target_deinit);
8735
8736 createRuntimeFunctionCall(Callee: Fn, Args: {});
8737
8738 if (!TeamsReductionDataSize)
8739 return;
8740
8741 Function *Kernel = Builder.GetInsertBlock()->getParent();
8742 // We need to strip the debug prefix to get the correct kernel name.
8743 StringRef KernelName = Kernel->getName();
8744 const std::string DebugPrefix = "_debug__";
8745 if (KernelName.ends_with(Suffix: DebugPrefix))
8746 KernelName = KernelName.drop_back(N: DebugPrefix.length());
8747 auto *KernelEnvironmentGV =
8748 M.getNamedGlobal(Name: (KernelName + "_kernel_environment").str());
8749 assert(KernelEnvironmentGV && "Expected kernel environment global\n");
8750 auto *KernelEnvironmentInitializer = KernelEnvironmentGV->getInitializer();
8751 auto *NewInitializer = ConstantFoldInsertValueInstruction(
8752 Agg: KernelEnvironmentInitializer,
8753 Val: ConstantInt::get(Ty: Int32, V: TeamsReductionDataSize), Idxs: {0, 7});
8754 KernelEnvironmentGV->setInitializer(NewInitializer);
8755}
8756
8757static void updateNVPTXAttr(Function &Kernel, StringRef Name, int32_t Value,
8758 bool Min) {
8759 if (Kernel.hasFnAttribute(Kind: Name)) {
8760 int32_t OldLimit = Kernel.getFnAttributeAsParsedInteger(Kind: Name);
8761 Value = Min ? std::min(a: OldLimit, b: Value) : std::max(a: OldLimit, b: Value);
8762 }
8763 Kernel.addFnAttr(Kind: Name, Val: llvm::utostr(X: Value));
8764}
8765
8766std::pair<int32_t, int32_t>
8767OpenMPIRBuilder::readThreadBoundsForKernel(const Triple &T, Function &Kernel) {
8768 int32_t ThreadLimit =
8769 Kernel.getFnAttributeAsParsedInteger(Kind: "omp_target_thread_limit");
8770
8771 if (T.isAMDGPU()) {
8772 const auto &Attr = Kernel.getFnAttribute(Kind: "amdgpu-flat-work-group-size");
8773 if (!Attr.isValid() || !Attr.isStringAttribute())
8774 return {0, ThreadLimit};
8775 auto [LBStr, UBStr] = Attr.getValueAsString().split(Separator: ',');
8776 int32_t LB, UB;
8777 if (!llvm::to_integer(S: UBStr, Num&: UB, Base: 10))
8778 return {0, ThreadLimit};
8779 UB = ThreadLimit ? std::min(a: ThreadLimit, b: UB) : UB;
8780 if (!llvm::to_integer(S: LBStr, Num&: LB, Base: 10))
8781 return {0, UB};
8782 return {LB, UB};
8783 }
8784
8785 if (Kernel.hasFnAttribute(Kind: NVVMAttr::MaxNTID)) {
8786 int32_t UB = Kernel.getFnAttributeAsParsedInteger(Kind: NVVMAttr::MaxNTID);
8787 return {0, ThreadLimit ? std::min(a: ThreadLimit, b: UB) : UB};
8788 }
8789 return {0, ThreadLimit};
8790}
8791
8792void OpenMPIRBuilder::writeThreadBoundsForKernel(const Triple &T,
8793 Function &Kernel, int32_t LB,
8794 int32_t UB) {
8795 Kernel.addFnAttr(Kind: "omp_target_thread_limit", Val: std::to_string(val: UB));
8796
8797 if (T.isAMDGPU()) {
8798 Kernel.addFnAttr(Kind: "amdgpu-flat-work-group-size",
8799 Val: llvm::utostr(X: LB) + "," + llvm::utostr(X: UB));
8800 return;
8801 }
8802
8803 updateNVPTXAttr(Kernel, Name: NVVMAttr::MaxNTID, Value: UB, Min: true);
8804}
8805
8806std::pair<int32_t, int32_t>
8807OpenMPIRBuilder::readTeamBoundsForKernel(const Triple &, Function &Kernel) {
8808 // TODO: Read from backend annotations if available.
8809 return {0, Kernel.getFnAttributeAsParsedInteger(Kind: "omp_target_num_teams")};
8810}
8811
8812void OpenMPIRBuilder::writeTeamsForKernel(const Triple &T, Function &Kernel,
8813 int32_t LB, int32_t UB) {
8814 if (UB > 0) {
8815 if (T.isNVPTX())
8816 Kernel.addFnAttr(Kind: NVVMAttr::MaxClusterRank, Val: llvm::utostr(X: UB));
8817 if (T.isAMDGPU())
8818 Kernel.addFnAttr(Kind: "amdgpu-max-num-workgroups", Val: llvm::utostr(X: UB) + ",1,1");
8819 }
8820
8821 Kernel.addFnAttr(Kind: "omp_target_num_teams", Val: std::to_string(val: LB));
8822}
8823
8824void OpenMPIRBuilder::setOutlinedTargetRegionFunctionAttributes(
8825 Function *OutlinedFn) {
8826 if (Config.isTargetDevice()) {
8827 OutlinedFn->setLinkage(GlobalValue::WeakODRLinkage);
8828 // TODO: Determine if DSO local can be set to true.
8829 OutlinedFn->setDSOLocal(false);
8830 OutlinedFn->setVisibility(GlobalValue::ProtectedVisibility);
8831 if (T.isAMDGCN())
8832 OutlinedFn->setCallingConv(CallingConv::AMDGPU_KERNEL);
8833 else if (T.isNVPTX())
8834 OutlinedFn->setCallingConv(CallingConv::PTX_Kernel);
8835 else if (T.isSPIRV())
8836 OutlinedFn->setCallingConv(CallingConv::SPIR_KERNEL);
8837 }
8838}
8839
8840Constant *OpenMPIRBuilder::createOutlinedFunctionID(Function *OutlinedFn,
8841 StringRef EntryFnIDName) {
8842 if (Config.isTargetDevice()) {
8843 assert(OutlinedFn && "The outlined function must exist if embedded");
8844 return OutlinedFn;
8845 }
8846
8847 return new GlobalVariable(
8848 M, Builder.getInt8Ty(), /*isConstant=*/true, GlobalValue::WeakAnyLinkage,
8849 Constant::getNullValue(Ty: Builder.getInt8Ty()), EntryFnIDName);
8850}
8851
8852Constant *OpenMPIRBuilder::createTargetRegionEntryAddr(Function *OutlinedFn,
8853 StringRef EntryFnName) {
8854 if (OutlinedFn)
8855 return OutlinedFn;
8856
8857 assert(!M.getGlobalVariable(EntryFnName, true) &&
8858 "Named kernel already exists?");
8859 return new GlobalVariable(
8860 M, Builder.getInt8Ty(), /*isConstant=*/true, GlobalValue::InternalLinkage,
8861 Constant::getNullValue(Ty: Builder.getInt8Ty()), EntryFnName);
8862}
8863
8864Error OpenMPIRBuilder::emitTargetRegionFunction(
8865 TargetRegionEntryInfo &EntryInfo,
8866 FunctionGenCallback &GenerateFunctionCallback, bool IsOffloadEntry,
8867 Function *&OutlinedFn, Constant *&OutlinedFnID) {
8868
8869 SmallString<64> EntryFnName;
8870 OffloadInfoManager.getTargetRegionEntryFnName(Name&: EntryFnName, EntryInfo);
8871
8872 if (Config.isTargetDevice() || !Config.openMPOffloadMandatory()) {
8873 Expected<Function *> CBResult = GenerateFunctionCallback(EntryFnName);
8874 if (!CBResult)
8875 return CBResult.takeError();
8876 OutlinedFn = *CBResult;
8877 } else {
8878 OutlinedFn = nullptr;
8879 }
8880
8881 // If this target outline function is not an offload entry, we don't need to
8882 // register it. This may be in the case of a false if clause, or if there are
8883 // no OpenMP targets.
8884 if (!IsOffloadEntry)
8885 return Error::success();
8886
8887 std::string EntryFnIDName =
8888 Config.isTargetDevice()
8889 ? std::string(EntryFnName)
8890 : createPlatformSpecificName(Parts: {EntryFnName, "region_id"});
8891
8892 OutlinedFnID = registerTargetRegionFunction(EntryInfo, OutlinedFunction: OutlinedFn,
8893 EntryFnName, EntryFnIDName);
8894 return Error::success();
8895}
8896
8897Constant *OpenMPIRBuilder::registerTargetRegionFunction(
8898 TargetRegionEntryInfo &EntryInfo, Function *OutlinedFn,
8899 StringRef EntryFnName, StringRef EntryFnIDName) {
8900 if (OutlinedFn)
8901 setOutlinedTargetRegionFunctionAttributes(OutlinedFn);
8902 auto OutlinedFnID = createOutlinedFunctionID(OutlinedFn, EntryFnIDName);
8903 auto EntryAddr = createTargetRegionEntryAddr(OutlinedFn, EntryFnName);
8904 OffloadInfoManager.registerTargetRegionEntryInfo(
8905 EntryInfo, Addr: EntryAddr, ID: OutlinedFnID,
8906 Flags: OffloadEntriesInfoManager::OMPTargetRegionEntryTargetRegion);
8907 return OutlinedFnID;
8908}
8909
8910OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTargetData(
8911 const LocationDescription &Loc, InsertPointTy AllocaIP,
8912 InsertPointTy CodeGenIP, ArrayRef<BasicBlock *> DeallocBlocks,
8913 Value *DeviceID, Value *IfCond, TargetDataInfo &Info,
8914 GenMapInfoCallbackTy GenMapInfoCB, CustomMapperCallbackTy CustomMapperCB,
8915 omp::RuntimeFunction *MapperFunc,
8916 function_ref<InsertPointOrErrorTy(InsertPointTy CodeGenIP,
8917 BodyGenTy BodyGenType)>
8918 BodyGenCB,
8919 function_ref<void(unsigned int, Value *)> DeviceAddrCB, Value *SrcLocInfo) {
8920 if (!updateToLocation(Loc))
8921 return InsertPointTy();
8922
8923 Builder.restoreIP(IP: CodeGenIP);
8924
8925 bool IsStandAlone = !BodyGenCB;
8926 MapInfosTy *MapInfo;
8927 // Generate the code for the opening of the data environment. Capture all the
8928 // arguments of the runtime call by reference because they are used in the
8929 // closing of the region.
8930 auto BeginThenGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
8931 ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
8932 MapInfo = &GenMapInfoCB(Builder.saveIP());
8933 if (Error Err = emitOffloadingArrays(
8934 AllocaIP, CodeGenIP: Builder.saveIP(), CombinedInfo&: *MapInfo, Info, CustomMapperCB,
8935 /*IsNonContiguous=*/true, DeviceAddrCB))
8936 return Err;
8937
8938 TargetDataRTArgs RTArgs;
8939 emitOffloadingArraysArgument(Builder, RTArgs, Info);
8940
8941 // Emit the number of elements in the offloading arrays.
8942 Value *PointerNum = Builder.getInt32(C: Info.NumberOfPtrs);
8943
8944 // Source location for the ident struct
8945 if (!SrcLocInfo) {
8946 uint32_t SrcLocStrSize;
8947 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8948 SrcLocInfo = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8949 }
8950
8951 SmallVector<llvm::Value *, 13> OffloadingArgs = {
8952 SrcLocInfo, DeviceID,
8953 PointerNum, RTArgs.BasePointersArray,
8954 RTArgs.PointersArray, RTArgs.SizesArray,
8955 RTArgs.MapTypesArray, RTArgs.MapNamesArray,
8956 RTArgs.MappersArray};
8957
8958 if (IsStandAlone) {
8959 assert(MapperFunc && "MapperFunc missing for standalone target data");
8960
8961 auto TaskBodyCB = [&](Value *, Value *,
8962 IRBuilderBase::InsertPoint) -> Error {
8963 if (Info.HasNoWait) {
8964 OffloadingArgs.append(IL: {llvm::Constant::getNullValue(Ty: Int32),
8965 llvm::Constant::getNullValue(Ty: VoidPtr),
8966 llvm::Constant::getNullValue(Ty: Int32),
8967 llvm::Constant::getNullValue(Ty: VoidPtr)});
8968 }
8969
8970 createRuntimeFunctionCall(Callee: getOrCreateRuntimeFunctionPtr(FnID: *MapperFunc),
8971 Args: OffloadingArgs);
8972
8973 if (Info.HasNoWait) {
8974 BasicBlock *OffloadContBlock =
8975 BasicBlock::Create(Context&: Builder.getContext(), Name: "omp_offload.cont");
8976 Function *CurFn = Builder.GetInsertBlock()->getParent();
8977 emitBlock(BB: OffloadContBlock, CurFn, /*IsFinished=*/true);
8978 Builder.restoreIP(IP: Builder.saveIP());
8979 }
8980 return Error::success();
8981 };
8982
8983 bool RequiresOuterTargetTask = Info.HasNoWait;
8984 if (!RequiresOuterTargetTask)
8985 cantFail(Err: TaskBodyCB(/*DeviceID=*/nullptr, /*RTLoc=*/nullptr,
8986 /*TargetTaskAllocaIP=*/{}));
8987 else
8988 cantFail(ValOrErr: emitTargetTask(TaskBodyCB, DeviceID, RTLoc: SrcLocInfo, AllocaIP,
8989 /*Dependencies=*/{}, RTArgs, HasNoWait: Info.HasNoWait));
8990 } else {
8991 Function *BeginMapperFunc = getOrCreateRuntimeFunctionPtr(
8992 FnID: omp::OMPRTL___tgt_target_data_begin_mapper);
8993
8994 createRuntimeFunctionCall(Callee: BeginMapperFunc, Args: OffloadingArgs);
8995
8996 for (auto DeviceMap : Info.DevicePtrInfoMap) {
8997 if (isa<AllocaInst>(Val: DeviceMap.second.second)) {
8998 auto *LI =
8999 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: DeviceMap.second.first);
9000 Builder.CreateStore(Val: LI, Ptr: DeviceMap.second.second);
9001 }
9002 }
9003
9004 // If device pointer privatization is required, emit the body of the
9005 // region here. It will have to be duplicated: with and without
9006 // privatization.
9007 InsertPointOrErrorTy AfterIP =
9008 BodyGenCB(Builder.saveIP(), BodyGenTy::Priv);
9009 if (!AfterIP)
9010 return AfterIP.takeError();
9011 Builder.restoreIP(IP: *AfterIP);
9012 }
9013 return Error::success();
9014 };
9015
9016 // If we need device pointer privatization, we need to emit the body of the
9017 // region with no privatization in the 'else' branch of the conditional.
9018 // Otherwise, we don't have to do anything.
9019 auto BeginElseGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
9020 ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
9021 InsertPointOrErrorTy AfterIP =
9022 BodyGenCB(Builder.saveIP(), BodyGenTy::DupNoPriv);
9023 if (!AfterIP)
9024 return AfterIP.takeError();
9025 Builder.restoreIP(IP: *AfterIP);
9026 return Error::success();
9027 };
9028
9029 // Generate code for the closing of the data region.
9030 auto EndThenGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
9031 ArrayRef<BasicBlock *> DeallocBlocks) {
9032 TargetDataRTArgs RTArgs;
9033 Info.EmitDebug = !MapInfo->Names.empty();
9034 emitOffloadingArraysArgument(Builder, RTArgs, Info, /*ForEndCall=*/true);
9035
9036 // Emit the number of elements in the offloading arrays.
9037 Value *PointerNum = Builder.getInt32(C: Info.NumberOfPtrs);
9038
9039 // Source location for the ident struct
9040 if (!SrcLocInfo) {
9041 uint32_t SrcLocStrSize;
9042 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
9043 SrcLocInfo = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
9044 }
9045
9046 Value *OffloadingArgs[] = {SrcLocInfo, DeviceID,
9047 PointerNum, RTArgs.BasePointersArray,
9048 RTArgs.PointersArray, RTArgs.SizesArray,
9049 RTArgs.MapTypesArray, RTArgs.MapNamesArray,
9050 RTArgs.MappersArray};
9051 Function *EndMapperFunc =
9052 getOrCreateRuntimeFunctionPtr(FnID: omp::OMPRTL___tgt_target_data_end_mapper);
9053
9054 createRuntimeFunctionCall(Callee: EndMapperFunc, Args: OffloadingArgs);
9055 return Error::success();
9056 };
9057
9058 // We don't have to do anything to close the region if the if clause evaluates
9059 // to false.
9060 auto EndElseGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
9061 ArrayRef<BasicBlock *> DeallocBlocks) {
9062 return Error::success();
9063 };
9064
9065 Error Err = [&]() -> Error {
9066 if (BodyGenCB) {
9067 Error Err = [&]() {
9068 if (IfCond)
9069 return emitIfClause(Cond: IfCond, ThenGen: BeginThenGen, ElseGen: BeginElseGen, AllocaIP);
9070 return BeginThenGen(AllocaIP, Builder.saveIP(), DeallocBlocks);
9071 }();
9072
9073 if (Err)
9074 return Err;
9075
9076 // If we don't require privatization of device pointers, we emit the body
9077 // in between the runtime calls. This avoids duplicating the body code.
9078 InsertPointOrErrorTy AfterIP =
9079 BodyGenCB(Builder.saveIP(), BodyGenTy::NoPriv);
9080 if (!AfterIP)
9081 return AfterIP.takeError();
9082 restoreIPandDebugLoc(Builder, IP: *AfterIP);
9083
9084 if (IfCond)
9085 return emitIfClause(Cond: IfCond, ThenGen: EndThenGen, ElseGen: EndElseGen, AllocaIP);
9086 return EndThenGen(AllocaIP, Builder.saveIP(), DeallocBlocks);
9087 }
9088 if (IfCond)
9089 return emitIfClause(Cond: IfCond, ThenGen: BeginThenGen, ElseGen: EndElseGen, AllocaIP);
9090 return BeginThenGen(AllocaIP, Builder.saveIP(), DeallocBlocks);
9091 }();
9092
9093 if (Err)
9094 return Err;
9095
9096 return Builder.saveIP();
9097}
9098
9099FunctionCallee
9100OpenMPIRBuilder::createForStaticInitFunction(unsigned IVSize, bool IVSigned,
9101 bool IsGPUDistribute) {
9102 assert((IVSize == 32 || IVSize == 64) &&
9103 "IV size is not compatible with the omp runtime");
9104 RuntimeFunction Name;
9105 if (IsGPUDistribute)
9106 Name = IVSize == 32
9107 ? (IVSigned ? omp::OMPRTL___kmpc_distribute_static_init_4
9108 : omp::OMPRTL___kmpc_distribute_static_init_4u)
9109 : (IVSigned ? omp::OMPRTL___kmpc_distribute_static_init_8
9110 : omp::OMPRTL___kmpc_distribute_static_init_8u);
9111 else
9112 Name = IVSize == 32 ? (IVSigned ? omp::OMPRTL___kmpc_for_static_init_4
9113 : omp::OMPRTL___kmpc_for_static_init_4u)
9114 : (IVSigned ? omp::OMPRTL___kmpc_for_static_init_8
9115 : omp::OMPRTL___kmpc_for_static_init_8u);
9116
9117 return getOrCreateRuntimeFunction(M, FnID: Name);
9118}
9119
9120FunctionCallee OpenMPIRBuilder::createDispatchInitFunction(unsigned IVSize,
9121 bool IVSigned) {
9122 assert((IVSize == 32 || IVSize == 64) &&
9123 "IV size is not compatible with the omp runtime");
9124 RuntimeFunction Name = IVSize == 32
9125 ? (IVSigned ? omp::OMPRTL___kmpc_dispatch_init_4
9126 : omp::OMPRTL___kmpc_dispatch_init_4u)
9127 : (IVSigned ? omp::OMPRTL___kmpc_dispatch_init_8
9128 : omp::OMPRTL___kmpc_dispatch_init_8u);
9129
9130 return getOrCreateRuntimeFunction(M, FnID: Name);
9131}
9132
9133FunctionCallee OpenMPIRBuilder::createDispatchNextFunction(unsigned IVSize,
9134 bool IVSigned) {
9135 assert((IVSize == 32 || IVSize == 64) &&
9136 "IV size is not compatible with the omp runtime");
9137 RuntimeFunction Name = IVSize == 32
9138 ? (IVSigned ? omp::OMPRTL___kmpc_dispatch_next_4
9139 : omp::OMPRTL___kmpc_dispatch_next_4u)
9140 : (IVSigned ? omp::OMPRTL___kmpc_dispatch_next_8
9141 : omp::OMPRTL___kmpc_dispatch_next_8u);
9142
9143 return getOrCreateRuntimeFunction(M, FnID: Name);
9144}
9145
9146FunctionCallee OpenMPIRBuilder::createDispatchFiniFunction(unsigned IVSize,
9147 bool IVSigned) {
9148 assert((IVSize == 32 || IVSize == 64) &&
9149 "IV size is not compatible with the omp runtime");
9150 RuntimeFunction Name = IVSize == 32
9151 ? (IVSigned ? omp::OMPRTL___kmpc_dispatch_fini_4
9152 : omp::OMPRTL___kmpc_dispatch_fini_4u)
9153 : (IVSigned ? omp::OMPRTL___kmpc_dispatch_fini_8
9154 : omp::OMPRTL___kmpc_dispatch_fini_8u);
9155
9156 return getOrCreateRuntimeFunction(M, FnID: Name);
9157}
9158
9159FunctionCallee OpenMPIRBuilder::createDispatchDeinitFunction() {
9160 return getOrCreateRuntimeFunction(M, FnID: omp::OMPRTL___kmpc_dispatch_deinit);
9161}
9162
9163static void FixupDebugInfoForOutlinedFunction(
9164 OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, Function *Func,
9165 DenseMap<Value *, std::tuple<Value *, unsigned>> &ValueReplacementMap) {
9166
9167 DISubprogram *NewSP = Func->getSubprogram();
9168 if (!NewSP)
9169 return;
9170
9171 SmallDenseMap<DILocalVariable *, DILocalVariable *> RemappedVariables;
9172
9173 auto GetUpdatedDIVariable = [&](DILocalVariable *OldVar, unsigned arg) {
9174 DILocalVariable *&NewVar = RemappedVariables[OldVar];
9175 // Only use cached variable if the arg number matches. This is important
9176 // so that DIVariable created for privatized variables are not discarded.
9177 if (NewVar && (arg == NewVar->getArg()))
9178 return NewVar;
9179
9180 NewVar = llvm::DILocalVariable::get(
9181 Context&: Builder.getContext(), Scope: OldVar->getScope(), Name: OldVar->getName(),
9182 File: OldVar->getFile(), Line: OldVar->getLine(), Type: OldVar->getType(), Arg: arg,
9183 Flags: OldVar->getFlags(), AlignInBits: OldVar->getAlignInBits(), Annotations: OldVar->getAnnotations());
9184 return NewVar;
9185 };
9186
9187 auto UpdateDebugRecord = [&](auto *DR) {
9188 DILocalVariable *OldVar = DR->getVariable();
9189 unsigned ArgNo = 0;
9190 for (auto Loc : DR->location_ops()) {
9191 auto Iter = ValueReplacementMap.find(Loc);
9192 if (Iter != ValueReplacementMap.end()) {
9193 DR->replaceVariableLocationOp(Loc, std::get<0>(Iter->second));
9194 ArgNo = std::get<1>(Iter->second) + 1;
9195 }
9196 }
9197 if (ArgNo != 0)
9198 DR->setVariable(GetUpdatedDIVariable(OldVar, ArgNo));
9199 };
9200
9201 SmallVector<DbgVariableRecord *, 4> DVRsToDelete;
9202 auto MoveDebugRecordToCorrectBlock = [&](DbgVariableRecord *DVR) {
9203 if (DVR->getNumVariableLocationOps() != 1u) {
9204 DVR->setKillLocation();
9205 return;
9206 }
9207 Value *Loc = DVR->getVariableLocationOp(OpIdx: 0u);
9208 BasicBlock *CurBB = DVR->getParent();
9209 BasicBlock *RequiredBB = nullptr;
9210
9211 if (Instruction *LocInst = dyn_cast<Instruction>(Val: Loc))
9212 RequiredBB = LocInst->getParent();
9213 else if (isa<llvm::Argument>(Val: Loc))
9214 RequiredBB = &DVR->getFunction()->getEntryBlock();
9215
9216 if (RequiredBB && RequiredBB != CurBB) {
9217 assert(!RequiredBB->empty());
9218 RequiredBB->insertDbgRecordBefore(DR: DVR->clone(),
9219 Here: RequiredBB->back().getIterator());
9220 DVRsToDelete.push_back(Elt: DVR);
9221 }
9222 };
9223
9224 // The location and scope of variable intrinsics and records still point to
9225 // the parent function of the target region. Update them.
9226 for (Instruction &I : instructions(F: Func)) {
9227 assert(!isa<llvm::DbgVariableIntrinsic>(&I) &&
9228 "Unexpected debug intrinsic");
9229 for (DbgVariableRecord &DVR : filterDbgVars(R: I.getDbgRecordRange())) {
9230 UpdateDebugRecord(&DVR);
9231 MoveDebugRecordToCorrectBlock(&DVR);
9232 }
9233 }
9234 for (auto *DVR : DVRsToDelete)
9235 DVR->getMarker()->MarkedInstr->dropOneDbgRecord(I: DVR);
9236 // An extra argument is passed to the device. Create the debug data for it.
9237 if (OMPBuilder.Config.isTargetDevice()) {
9238 DICompileUnit *CU = NewSP->getUnit();
9239 Module *M = Func->getParent();
9240 DIBuilder DB(*M, true, CU);
9241 DIType *VoidPtrTy =
9242 DB.createQualifiedType(Tag: dwarf::DW_TAG_pointer_type, FromTy: nullptr);
9243 unsigned ArgNo = Func->arg_size();
9244 DILocalVariable *Var = DB.createParameterVariable(
9245 Scope: NewSP, Name: "dyn_ptr", ArgNo, File: NewSP->getFile(), /*LineNo=*/0, Ty: VoidPtrTy,
9246 /*AlwaysPreserve=*/false, Flags: DINode::DIFlags::FlagArtificial);
9247 auto Loc = DILocation::get(Context&: Func->getContext(), Line: 0, Column: 0, Scope: NewSP, InlinedAt: 0);
9248 Argument *LastArg = Func->getArg(i: Func->arg_size() - 1);
9249 DB.insertDeclare(Storage: LastArg, VarInfo: Var, Expr: DB.createExpression(), DL: Loc,
9250 InsertAtEnd: &(*Func->begin()));
9251 }
9252}
9253
9254static Value *removeASCastIfPresent(Value *V) {
9255 if (Operator::getOpcode(V) == Instruction::AddrSpaceCast)
9256 return cast<Operator>(Val: V)->getOperand(i: 0);
9257 return V;
9258}
9259
9260static Expected<Function *> createOutlinedFunction(
9261 OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder,
9262 const OpenMPIRBuilder::TargetKernelDefaultAttrs &DefaultAttrs,
9263 StringRef FuncName, SmallVectorImpl<Value *> &Inputs,
9264 OpenMPIRBuilder::TargetBodyGenCallbackTy &CBFunc,
9265 OpenMPIRBuilder::TargetGenArgAccessorsCallbackTy &ArgAccessorFuncCB,
9266 DebugLoc OutlinedFnLoc) {
9267 SmallVector<Type *> ParameterTypes;
9268 if (OMPBuilder.Config.isTargetDevice()) {
9269 // All parameters to target devices are passed as pointers
9270 // or i64. This assumes 64-bit address spaces/pointers.
9271 for (auto &Arg : Inputs)
9272 ParameterTypes.push_back(Elt: Arg->getType()->isPointerTy()
9273 ? Arg->getType()
9274 : Type::getInt64Ty(C&: Builder.getContext()));
9275 } else {
9276 for (auto &Arg : Inputs)
9277 ParameterTypes.push_back(Elt: Arg->getType());
9278 }
9279
9280 // The implicit dyn_ptr argument is always the last parameter on both host
9281 // and device so the argument counts match without runtime manipulation.
9282 auto *PtrTy = PointerType::getUnqual(C&: Builder.getContext());
9283 ParameterTypes.push_back(Elt: PtrTy);
9284
9285 auto BB = Builder.GetInsertBlock();
9286 auto M = BB->getModule();
9287 auto FuncType = FunctionType::get(Result: Builder.getVoidTy(), Params: ParameterTypes,
9288 /*isVarArg*/ false);
9289 auto Func =
9290 Function::Create(Ty: FuncType, Linkage: GlobalValue::InternalLinkage, N: FuncName, M);
9291
9292 // Forward target-cpu and target-features function attributes from the
9293 // original function to the new outlined function.
9294 Function *ParentFn = Builder.GetInsertBlock()->getParent();
9295
9296 auto TargetCpuAttr = ParentFn->getFnAttribute(Kind: "target-cpu");
9297 if (TargetCpuAttr.isStringAttribute())
9298 Func->addFnAttr(Attr: TargetCpuAttr);
9299
9300 auto TargetFeaturesAttr = ParentFn->getFnAttribute(Kind: "target-features");
9301 if (TargetFeaturesAttr.isStringAttribute())
9302 Func->addFnAttr(Attr: TargetFeaturesAttr);
9303
9304 if (OMPBuilder.Config.isTargetDevice()) {
9305 Value *ExecMode =
9306 OMPBuilder.emitKernelExecutionMode(KernelName: FuncName, Mode: DefaultAttrs.ExecFlags);
9307 OMPBuilder.emitUsed(Name: "llvm.compiler.used", List: {ExecMode});
9308 }
9309
9310 // Save insert point.
9311 IRBuilder<>::InsertPointGuard IPG(Builder);
9312 // We will generate the entries in the outlined function but the debug
9313 // location is still pointing to the parent function, which is the wrong
9314 // scope. OutlinedFnLoc, when the caller provides one, is the same source
9315 // position scoped to the subprogram that will be attached to the outlined
9316 // function, so it is what everything emitted below needs.
9317 Builder.SetCurrentDebugLocation(OutlinedFnLoc);
9318
9319 // Generate the region into the function.
9320 BasicBlock *EntryBB = BasicBlock::Create(Context&: Builder.getContext(), Name: "entry", Parent: Func);
9321 Builder.SetInsertPoint(EntryBB);
9322
9323 // Insert target init call in the device compilation pass.
9324 if (OMPBuilder.Config.isTargetDevice())
9325 Builder.restoreIP(IP: OMPBuilder.createTargetInit(Loc: Builder, Attrs: DefaultAttrs));
9326
9327 BasicBlock *UserCodeEntryBB = Builder.GetInsertBlock();
9328
9329 // As we embed the user code in the middle of our target region after we
9330 // generate entry code, we must move what allocas we can into the entry
9331 // block to avoid possible breaking optimisations for device
9332 if (OMPBuilder.Config.isTargetDevice())
9333 OMPBuilder.ConstantAllocaRaiseCandidates.emplace_back(Args&: Func);
9334
9335 BasicBlock *ExitBB = splitBB(Builder, /*CreateBranch=*/true, Name: "target.exit");
9336 BasicBlock *OutlinedBodyBB =
9337 splitBB(Builder, /*CreateBranch=*/true, Name: "outlined.body");
9338 llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = CBFunc(
9339 Builder.saveIP(),
9340 OpenMPIRBuilder::InsertPointTy(OutlinedBodyBB, OutlinedBodyBB->begin()),
9341 ExitBB);
9342 if (!AfterIP)
9343 return AfterIP.takeError();
9344 Builder.SetInsertPoint(ExitBB);
9345 // The body callback builds the body with its own IRBuilder and cannot reach
9346 // this one directly. But a body holding another OpenMP construct, a nested
9347 // parallel say, calls OpenMPIRBuilder::createParallel, and that can leave
9348 // this Builder pointing at the wrong debug location, or at none at all. The
9349 // epilogue below belongs to the target construct rather than to whatever the
9350 // body emitted last, so re-establish the location the prologue was emitted
9351 // with.
9352 Builder.SetCurrentDebugLocation(OutlinedFnLoc);
9353
9354 // Insert target deinit call in the device compilation pass.
9355 if (OMPBuilder.Config.isTargetDevice())
9356 OMPBuilder.createTargetDeinit(Loc: Builder);
9357
9358 // Insert return instruction.
9359 Builder.CreateRetVoid();
9360
9361 // New Alloca IP at entry point of created device function.
9362 Builder.SetInsertPoint(EntryBB->getFirstNonPHIIt());
9363 auto AllocaIP = Builder.saveIP();
9364
9365 Builder.SetInsertPoint(UserCodeEntryBB->getFirstNonPHIOrDbg());
9366
9367 // Do not include the artificial dyn_ptr argument.
9368 const auto &ArgRange = make_range(x: Func->arg_begin(), y: Func->arg_end() - 1);
9369
9370 DenseMap<Value *, std::tuple<Value *, unsigned>> ValueReplacementMap;
9371
9372 auto ReplaceValue = [](Value *Input, Value *InputCopy, Function *Func) {
9373 // Things like GEP's can come in the form of Constants. Constants and
9374 // ConstantExpr's do not have access to the knowledge of what they're
9375 // contained in, so we must dig a little to find an instruction so we
9376 // can tell if they're used inside of the function we're outlining. We
9377 // also replace the original constant expression with a new instruction
9378 // equivalent; an instruction as it allows easy modification in the
9379 // following loop, as we can now know the constant (instruction) is
9380 // owned by our target function and replaceUsesOfWith can now be invoked
9381 // on it (cannot do this with constants it seems). A brand new one also
9382 // allows us to be cautious as it is perhaps possible the old expression
9383 // was used inside of the function but exists and is used externally
9384 // (unlikely by the nature of a Constant, but still).
9385 // NOTE: We cannot remove dead constants that have been rewritten to
9386 // instructions at this stage, we run the risk of breaking later lowering
9387 // by doing so as we could still be in the process of lowering the module
9388 // from MLIR to LLVM-IR and the MLIR lowering may still require the original
9389 // constants we have created rewritten versions of.
9390 if (auto *Const = dyn_cast<Constant>(Val: Input))
9391 convertUsersOfConstantsToInstructions(Consts: Const, RestrictToFunc: Func, RemoveDeadConstants: false);
9392
9393 // Collect users before iterating over them to avoid invalidating the
9394 // iteration in case a user uses Input more than once (e.g. a call
9395 // instruction).
9396 SetVector<User *> Users(Input->users().begin(), Input->users().end());
9397 // Collect all the instructions
9398 for (User *User : make_early_inc_range(Range&: Users))
9399 if (auto *Instr = dyn_cast<Instruction>(Val: User))
9400 if (Instr->getFunction() == Func)
9401 Instr->replaceUsesOfWith(From: Input, To: InputCopy);
9402 };
9403
9404 SmallVector<std::pair<Value *, Value *>> DeferredReplacement;
9405
9406 // Rewrite uses of input valus to parameters.
9407 for (auto InArg : zip(t&: Inputs, u: ArgRange)) {
9408 Value *Input = std::get<0>(t&: InArg);
9409 Argument &Arg = std::get<1>(t&: InArg);
9410 Value *InputCopy = nullptr;
9411
9412 llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = ArgAccessorFuncCB(
9413 Arg, Input, InputCopy, AllocaIP, Builder.saveIP(),
9414 OpenMPIRBuilder::InsertPointTy(ExitBB, ExitBB->begin()));
9415 if (!AfterIP)
9416 return AfterIP.takeError();
9417 Builder.restoreIP(IP: *AfterIP);
9418 ValueReplacementMap[Input] = std::make_tuple(args&: InputCopy, args: Arg.getArgNo());
9419
9420 // In certain cases a Global may be set up for replacement, however, this
9421 // Global may be used in multiple arguments to the kernel, just segmented
9422 // apart, for example, if we have a global array, that is sectioned into
9423 // multiple mappings (technically not legal in OpenMP, but there is a case
9424 // in Fortran for Common Blocks where this is neccesary), we will end up
9425 // with GEP's into this array inside the kernel, that refer to the Global
9426 // but are technically separate arguments to the kernel for all intents and
9427 // purposes. If we have mapped a segment that requires a GEP into the 0-th
9428 // index, it will fold into an referal to the Global, if we then encounter
9429 // this folded GEP during replacement all of the references to the
9430 // Global in the kernel will be replaced with the argument we have generated
9431 // that corresponds to it, including any other GEP's that refer to the
9432 // Global that may be other arguments. This will invalidate all of the other
9433 // preceding mapped arguments that refer to the same global that may be
9434 // separate segments. To prevent this, we defer global processing until all
9435 // other processing has been performed.
9436 if (llvm::isa<llvm::GlobalValue, llvm::GlobalObject, llvm::GlobalVariable>(
9437 Val: removeASCastIfPresent(V: Input))) {
9438 DeferredReplacement.push_back(Elt: std::make_pair(x&: Input, y&: InputCopy));
9439 continue;
9440 }
9441
9442 if (isa<ConstantData>(Val: Input))
9443 continue;
9444
9445 ReplaceValue(Input, InputCopy, Func);
9446 }
9447
9448 // Replace all of our deferred Input values, currently just Globals.
9449 for (auto Deferred : DeferredReplacement)
9450 ReplaceValue(std::get<0>(in&: Deferred), std::get<1>(in&: Deferred), Func);
9451
9452 FixupDebugInfoForOutlinedFunction(OMPBuilder, Builder, Func,
9453 ValueReplacementMap);
9454 return Func;
9455}
9456/// Given a task descriptor, TaskWithPrivates, return the pointer to the block
9457/// of pointers containing shared data between the parent task and the created
9458/// task.
9459static LoadInst *loadSharedDataFromTaskDescriptor(OpenMPIRBuilder &OMPIRBuilder,
9460 IRBuilderBase &Builder,
9461 Value *TaskWithPrivates,
9462 Type *TaskWithPrivatesTy) {
9463
9464 Type *TaskTy = OMPIRBuilder.Task;
9465 LLVMContext &Ctx = Builder.getContext();
9466 Value *TaskT =
9467 Builder.CreateStructGEP(Ty: TaskWithPrivatesTy, Ptr: TaskWithPrivates, Idx: 0);
9468 Value *Shareds = TaskT;
9469 // TaskWithPrivatesTy can be one of the following
9470 // 1. %struct.task_with_privates = type { %struct.kmp_task_ompbuilder_t,
9471 // %struct.privates }
9472 // 2. %struct.kmp_task_ompbuilder_t ;; This is simply TaskTy
9473 //
9474 // In the former case, that is when TaskWithPrivatesTy != TaskTy,
9475 // its first member has to be the task descriptor. TaskTy is the type of the
9476 // task descriptor. TaskT is the pointer to the task descriptor. Loading the
9477 // first member of TaskT, gives us the pointer to shared data.
9478 if (TaskWithPrivatesTy != TaskTy)
9479 Shareds = Builder.CreateStructGEP(Ty: TaskTy, Ptr: TaskT, Idx: 0);
9480 return Builder.CreateLoad(Ty: PointerType::getUnqual(C&: Ctx), Ptr: Shareds);
9481}
9482/// Create an entry point for a target task with the following.
9483/// It'll have the following signature
9484/// void @.omp_target_task_proxy_func(i32 %thread.id, ptr %task)
9485/// This function is called from emitTargetTask once the
9486/// code to launch the target kernel has been outlined already.
9487/// NumOffloadingArrays is the number of offloading arrays that we need to copy
9488/// into the task structure so that the deferred target task can access this
9489/// data even after the stack frame of the generating task has been rolled
9490/// back. Offloading arrays contain base pointers, pointers, sizes etc
9491/// of the data that the target kernel will access. These in effect are the
9492/// non-empty arrays of pointers held by OpenMPIRBuilder::TargetDataRTArgs.
9493static Function *emitTargetTaskProxyFunction(
9494 OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, CallInst *StaleCI,
9495 StructType *PrivatesTy, StructType *TaskWithPrivatesTy,
9496 const size_t NumOffloadingArrays, const int SharedArgsOperandNo) {
9497
9498 // If NumOffloadingArrays is non-zero, PrivatesTy better not be nullptr.
9499 // This is because PrivatesTy is the type of the structure in which
9500 // we pass the offloading arrays to the deferred target task.
9501 assert((!NumOffloadingArrays || PrivatesTy) &&
9502 "PrivatesTy cannot be nullptr when there are offloadingArrays"
9503 "to privatize");
9504
9505 Module &M = OMPBuilder.M;
9506 // KernelLaunchFunction is the target launch function, i.e.
9507 // the function that sets up kernel arguments and calls
9508 // __tgt_target_kernel to launch the kernel on the device.
9509 //
9510 Function *KernelLaunchFunction = StaleCI->getCalledFunction();
9511
9512 // StaleCI is the CallInst which is the call to the outlined
9513 // target kernel launch function. If there are local live-in values
9514 // that the outlined function uses then these are aggregated into a structure
9515 // which is passed as the second argument. If there are no local live-in
9516 // values or if all values used by the outlined kernel are global variables,
9517 // then there's only one argument, the threadID. So, StaleCI can be
9518 //
9519 // %structArg = alloca { ptr, ptr }, align 8
9520 // %gep_ = getelementptr { ptr, ptr }, ptr %structArg, i32 0, i32 0
9521 // store ptr %20, ptr %gep_, align 8
9522 // %gep_8 = getelementptr { ptr, ptr }, ptr %structArg, i32 0, i32 1
9523 // store ptr %21, ptr %gep_8, align 8
9524 // call void @_QQmain..omp_par.1(i32 %global.tid.val6, ptr %structArg)
9525 //
9526 // OR
9527 //
9528 // call void @_QQmain..omp_par.1(i32 %global.tid.val6)
9529 OpenMPIRBuilder::InsertPointTy IP(StaleCI->getParent(),
9530 StaleCI->getIterator());
9531
9532 LLVMContext &Ctx = StaleCI->getParent()->getContext();
9533
9534 Type *ThreadIDTy = Type::getInt32Ty(C&: Ctx);
9535 Type *TaskPtrTy = OMPBuilder.TaskPtr;
9536 [[maybe_unused]] Type *TaskTy = OMPBuilder.Task;
9537
9538 auto ProxyFnTy =
9539 FunctionType::get(Result: Builder.getVoidTy(), Params: {ThreadIDTy, TaskPtrTy},
9540 /* isVarArg */ false);
9541 auto ProxyFn = Function::Create(Ty: ProxyFnTy, Linkage: GlobalValue::InternalLinkage,
9542 N: ".omp_target_task_proxy_func", M);
9543 Value *ThreadId = ProxyFn->getArg(i: 0);
9544 Value *TaskWithPrivates = ProxyFn->getArg(i: 1);
9545 ThreadId->setName("thread.id");
9546 TaskWithPrivates->setName("task");
9547
9548 bool HasShareds = SharedArgsOperandNo > 0;
9549 bool HasOffloadingArrays = NumOffloadingArrays > 0;
9550 IRBuilder<>::InsertPointGuard IPG(Builder);
9551 BasicBlock *EntryBB =
9552 BasicBlock::Create(Context&: Builder.getContext(), Name: "entry", Parent: ProxyFn);
9553 Builder.SetInsertPoint(EntryBB);
9554 Builder.SetCurrentDebugLocation(llvm::DebugLoc());
9555
9556 SmallVector<Value *> KernelLaunchArgs;
9557 KernelLaunchArgs.reserve(N: StaleCI->arg_size());
9558 KernelLaunchArgs.push_back(Elt: ThreadId);
9559
9560 if (HasOffloadingArrays) {
9561 assert(TaskTy != TaskWithPrivatesTy &&
9562 "If there are offloading arrays to pass to the target"
9563 "TaskTy cannot be the same as TaskWithPrivatesTy");
9564 (void)TaskTy;
9565 Value *Privates =
9566 Builder.CreateStructGEP(Ty: TaskWithPrivatesTy, Ptr: TaskWithPrivates, Idx: 1);
9567 for (unsigned int i = 0; i < NumOffloadingArrays; ++i)
9568 KernelLaunchArgs.push_back(
9569 Elt: Builder.CreateStructGEP(Ty: PrivatesTy, Ptr: Privates, Idx: i));
9570 }
9571
9572 if (HasShareds) {
9573 auto *ArgStructAlloca =
9574 dyn_cast<AllocaInst>(Val: StaleCI->getArgOperand(i: SharedArgsOperandNo));
9575 assert(ArgStructAlloca &&
9576 "Unable to find the alloca instruction corresponding to arguments "
9577 "for extracted function");
9578 auto *ArgStructType = cast<StructType>(Val: ArgStructAlloca->getAllocatedType());
9579 std::optional<TypeSize> ArgAllocSize =
9580 ArgStructAlloca->getAllocationSize(DL: M.getDataLayout());
9581 assert(ArgStructType && ArgAllocSize &&
9582 "Unable to determine size of arguments for extracted function");
9583 uint64_t StructSize = ArgAllocSize->getFixedValue();
9584
9585 AllocaInst *NewArgStructAlloca =
9586 Builder.CreateAlloca(Ty: ArgStructType, ArraySize: nullptr, Name: "structArg");
9587
9588 Value *SharedsSize = Builder.getInt64(C: StructSize);
9589
9590 LoadInst *LoadShared = loadSharedDataFromTaskDescriptor(
9591 OMPIRBuilder&: OMPBuilder, Builder, TaskWithPrivates, TaskWithPrivatesTy);
9592
9593 Builder.CreateMemCpy(
9594 Dst: NewArgStructAlloca, DstAlign: NewArgStructAlloca->getAlign(), Src: LoadShared,
9595 SrcAlign: LoadShared->getPointerAlignment(DL: M.getDataLayout()), Size: SharedsSize);
9596 KernelLaunchArgs.push_back(Elt: NewArgStructAlloca);
9597 }
9598 OMPBuilder.createRuntimeFunctionCall(Callee: KernelLaunchFunction, Args: KernelLaunchArgs);
9599 Builder.CreateRetVoid();
9600 return ProxyFn;
9601}
9602static Type *getOffloadingArrayType(Value *V) {
9603
9604 if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: V))
9605 return GEP->getSourceElementType();
9606 if (auto *Alloca = dyn_cast<AllocaInst>(Val: V))
9607 return Alloca->getAllocatedType();
9608
9609 llvm_unreachable("Unhandled Instruction type");
9610 return nullptr;
9611}
9612// This function returns a struct that has at most two members.
9613// The first member is always %struct.kmp_task_ompbuilder_t, that is the task
9614// descriptor. The second member, if needed, is a struct containing arrays
9615// that need to be passed to the offloaded target kernel. For example,
9616// if .offload_baseptrs, .offload_ptrs and .offload_sizes have to be passed to
9617// the target kernel and their types are [3 x ptr], [3 x ptr] and [3 x i64]
9618// respectively, then the types created by this function are
9619//
9620// %struct.privates = type { [3 x ptr], [3 x ptr], [3 x i64] }
9621// %struct.task_with_privates = type { %struct.kmp_task_ompbuilder_t,
9622// %struct.privates }
9623// %struct.task_with_privates is returned by this function.
9624// If there aren't any offloading arrays to pass to the target kernel,
9625// %struct.kmp_task_ompbuilder_t is returned.
9626static StructType *
9627createTaskWithPrivatesTy(OpenMPIRBuilder &OMPIRBuilder,
9628 ArrayRef<Value *> OffloadingArraysToPrivatize) {
9629
9630 if (OffloadingArraysToPrivatize.empty())
9631 return OMPIRBuilder.Task;
9632
9633 SmallVector<Type *, 4> StructFieldTypes;
9634 for (Value *V : OffloadingArraysToPrivatize) {
9635 assert(V->getType()->isPointerTy() &&
9636 "Expected pointer to array to privatize. Got a non-pointer value "
9637 "instead");
9638 Type *ArrayTy = getOffloadingArrayType(V);
9639 assert(ArrayTy && "ArrayType cannot be nullptr");
9640 StructFieldTypes.push_back(Elt: ArrayTy);
9641 }
9642 StructType *PrivatesStructTy =
9643 StructType::create(Elements: StructFieldTypes, Name: "struct.privates");
9644 return StructType::create(Elements: {OMPIRBuilder.Task, PrivatesStructTy},
9645 Name: "struct.task_with_privates");
9646}
9647static Error emitTargetOutlinedFunction(
9648 OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, bool IsOffloadEntry,
9649 TargetRegionEntryInfo &EntryInfo,
9650 const OpenMPIRBuilder::TargetKernelDefaultAttrs &DefaultAttrs,
9651 Function *&OutlinedFn, Constant *&OutlinedFnID,
9652 SmallVectorImpl<Value *> &Inputs,
9653 OpenMPIRBuilder::TargetBodyGenCallbackTy &CBFunc,
9654 OpenMPIRBuilder::TargetGenArgAccessorsCallbackTy &ArgAccessorFuncCB,
9655 DebugLoc OutlinedFnLoc) {
9656
9657 OpenMPIRBuilder::FunctionGenCallback &&GenerateOutlinedFunction =
9658 [&](StringRef EntryFnName) {
9659 return createOutlinedFunction(OMPBuilder, Builder, DefaultAttrs,
9660 FuncName: EntryFnName, Inputs, CBFunc,
9661 ArgAccessorFuncCB, OutlinedFnLoc);
9662 };
9663
9664 return OMPBuilder.emitTargetRegionFunction(
9665 EntryInfo, GenerateFunctionCallback&: GenerateOutlinedFunction, IsOffloadEntry, OutlinedFn,
9666 OutlinedFnID);
9667}
9668
9669OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitTargetTask(
9670 TargetTaskBodyCallbackTy TaskBodyCB, Value *DeviceID, Value *RTLoc,
9671 OpenMPIRBuilder::InsertPointTy AllocaIP,
9672 const DependenciesInfo &Dependencies, const TargetDataRTArgs &RTArgs,
9673 bool HasNoWait) {
9674
9675 // The following explains the code-gen scenario for the `target` directive. A
9676 // similar scneario is followed for other device-related directives (e.g.
9677 // `target enter data`) but in similar fashion since we only need to emit task
9678 // that encapsulates the proper runtime call.
9679 //
9680 // When we arrive at this function, the target region itself has been
9681 // outlined into the function OutlinedFn.
9682 // So at ths point, for
9683 // --------------------------------------------------------------
9684 // void user_code_that_offloads(...) {
9685 // omp target depend(..) map(from:a) map(to:b) private(i)
9686 // do i = 1, 10
9687 // a(i) = b(i) + n
9688 // }
9689 //
9690 // --------------------------------------------------------------
9691 //
9692 // we have
9693 //
9694 // --------------------------------------------------------------
9695 //
9696 // void user_code_that_offloads(...) {
9697 // %.offload_baseptrs = alloca [2 x ptr], align 8
9698 // %.offload_ptrs = alloca [2 x ptr], align 8
9699 // %.offload_mappers = alloca [2 x ptr], align 8
9700 // ;; target region has been outlined and now we need to
9701 // ;; offload to it via a target task.
9702 // }
9703 // void outlined_device_function(ptr a, ptr b, ptr n) {
9704 // n = *n_ptr;
9705 // do i = 1, 10
9706 // a(i) = b(i) + n
9707 // }
9708 //
9709 // We have to now do the following
9710 // (i) Make an offloading call to outlined_device_function using the OpenMP
9711 // RTL. See 'kernel_launch_function' in the pseudo code below. This is
9712 // emitted by emitKernelLaunch
9713 // (ii) Create a task entry point function that calls kernel_launch_function
9714 // and is the entry point for the target task. See
9715 // '@.omp_target_task_proxy_func in the pseudocode below.
9716 // (iii) Create a task with the task entry point created in (ii)
9717 //
9718 // That is we create the following
9719 // struct task_with_privates {
9720 // struct kmp_task_ompbuilder_t task_struct;
9721 // struct privates {
9722 // [2 x ptr] ; baseptrs
9723 // [2 x ptr] ; ptrs
9724 // [2 x i64] ; sizes
9725 // }
9726 // }
9727 // void user_code_that_offloads(...) {
9728 // %.offload_baseptrs = alloca [2 x ptr], align 8
9729 // %.offload_ptrs = alloca [2 x ptr], align 8
9730 // %.offload_sizes = alloca [2 x i64], align 8
9731 //
9732 // %structArg = alloca { ptr, ptr, ptr }, align 8
9733 // %strucArg[0] = a
9734 // %strucArg[1] = b
9735 // %strucArg[2] = &n
9736 //
9737 // target_task_with_privates = @__kmpc_omp_target_task_alloc(...,
9738 // sizeof(kmp_task_ompbuilder_t),
9739 // sizeof(structArg),
9740 // @.omp_target_task_proxy_func,
9741 // ...)
9742 // memcpy(target_task_with_privates->task_struct->shareds, %structArg,
9743 // sizeof(structArg))
9744 // memcpy(target_task_with_privates->privates->baseptrs,
9745 // offload_baseptrs, sizeof(offload_baseptrs)
9746 // memcpy(target_task_with_privates->privates->ptrs,
9747 // offload_ptrs, sizeof(offload_ptrs)
9748 // memcpy(target_task_with_privates->privates->sizes,
9749 // offload_sizes, sizeof(offload_sizes)
9750 // dependencies_array = ...
9751 // ;; if nowait not present
9752 // call @__kmpc_omp_wait_deps(..., dependencies_array)
9753 // call @__kmpc_omp_task_begin_if0(...)
9754 // call @ @.omp_target_task_proxy_func(i32 thread_id, ptr
9755 // %target_task_with_privates)
9756 // call @__kmpc_omp_task_complete_if0(...)
9757 // }
9758 //
9759 // define internal void @.omp_target_task_proxy_func(i32 %thread.id,
9760 // ptr %task) {
9761 // %structArg = alloca {ptr, ptr, ptr}
9762 // %task_ptr = getelementptr(%task, 0, 0)
9763 // %shared_data = load (getelementptr %task_ptr, 0, 0)
9764 // mempcy(%structArg, %shared_data, sizeof(%structArg))
9765 //
9766 // %offloading_arrays = getelementptr(%task, 0, 1)
9767 // %offload_baseptrs = getelementptr(%offloading_arrays, 0, 0)
9768 // %offload_ptrs = getelementptr(%offloading_arrays, 0, 1)
9769 // %offload_sizes = getelementptr(%offloading_arrays, 0, 2)
9770 // kernel_launch_function(%thread.id, %offload_baseptrs, %offload_ptrs,
9771 // %offload_sizes, %structArg)
9772 // }
9773 //
9774 // We need the proxy function because the signature of the task entry point
9775 // expected by kmpc_omp_task is always the same and will be different from
9776 // that of the kernel_launch function.
9777 //
9778 // kernel_launch_function is generated by emitKernelLaunch and has the
9779 // always_inline attribute. For this example, it'll look like so:
9780 // void kernel_launch_function(%thread_id, %offload_baseptrs, %offload_ptrs,
9781 // %offload_sizes, %structArg) alwaysinline {
9782 // %kernel_args = alloca %struct.__tgt_kernel_arguments, align 8
9783 // ; load aggregated data from %structArg
9784 // ; setup kernel_args using offload_baseptrs, offload_ptrs and
9785 // ; offload_sizes
9786 // call i32 @__tgt_target_kernel(...,
9787 // outlined_device_function,
9788 // ptr %kernel_args)
9789 // }
9790 // void outlined_device_function(ptr a, ptr b, ptr n) {
9791 // n = *n_ptr;
9792 // do i = 1, 10
9793 // a(i) = b(i) + n
9794 // }
9795 //
9796 BasicBlock *TargetTaskBodyBB =
9797 splitBB(Builder, /*CreateBranch=*/true, Name: "target.task.body");
9798 BasicBlock *TargetTaskAllocaBB =
9799 splitBB(Builder, /*CreateBranch=*/true, Name: "target.task.alloca");
9800
9801 InsertPointTy TargetTaskAllocaIP(TargetTaskAllocaBB,
9802 TargetTaskAllocaBB->begin());
9803 InsertPointTy TargetTaskBodyIP(TargetTaskBodyBB, TargetTaskBodyBB->begin());
9804
9805 auto OI = std::make_unique<OutlineInfo>();
9806 OI->EntryBB = TargetTaskAllocaBB;
9807 OI->OuterAllocBB = AllocaIP.getBlock();
9808
9809 // Add the thread ID argument.
9810 SmallVector<Instruction *, 4> ToBeDeleted;
9811 OI->ExcludeArgsFromAggregate.push_back(Elt: createFakeIntVal(
9812 Builder, OuterAllocaIP: AllocaIP, ToBeDeleted, InnerAllocaIP: TargetTaskAllocaIP, Name: "global.tid", AsPtr: false));
9813
9814 // Generate the task body which will subsequently be outlined.
9815 Builder.restoreIP(IP: TargetTaskBodyIP);
9816 if (Error Err = TaskBodyCB(DeviceID, RTLoc, TargetTaskAllocaIP))
9817 return Err;
9818
9819 // The outliner (CodeExtractor) extract a sequence or vector of blocks that
9820 // it is given. These blocks are enumerated by
9821 // OpenMPIRBuilder::OutlineInfo::collectBlocks which expects the OI.ExitBlock
9822 // to be outside the region. In other words, OI.ExitBlock is expected to be
9823 // the start of the region after the outlining. We used to set OI.ExitBlock
9824 // to the InsertBlock after TaskBodyCB is done. This is fine in most cases
9825 // except when the task body is a single basic block. In that case,
9826 // OI.ExitBlock is set to the single task body block and will get left out of
9827 // the outlining process. So, simply create a new empty block to which we
9828 // uncoditionally branch from where TaskBodyCB left off
9829 OI->ExitBB = BasicBlock::Create(Context&: Builder.getContext(), Name: "target.task.cont");
9830 emitBlock(BB: OI->ExitBB, CurFn: Builder.GetInsertBlock()->getParent(),
9831 /*IsFinished=*/true);
9832
9833 SmallVector<Value *, 2> OffloadingArraysToPrivatize;
9834 bool NeedsTargetTask = HasNoWait && DeviceID;
9835 if (NeedsTargetTask) {
9836 for (auto *V :
9837 {RTArgs.BasePointersArray, RTArgs.PointersArray, RTArgs.MappersArray,
9838 RTArgs.MapNamesArray, RTArgs.MapTypesArray, RTArgs.MapTypesArrayEnd,
9839 RTArgs.SizesArray}) {
9840 if (V && !isa<ConstantPointerNull, GlobalVariable>(Val: V)) {
9841 OffloadingArraysToPrivatize.push_back(Elt: V);
9842 OI->ExcludeArgsFromAggregate.push_back(Elt: V);
9843 }
9844 }
9845 }
9846 OI->PostOutlineCB = [this, ToBeDeleted, Dependencies, NeedsTargetTask,
9847 DeviceID, OffloadingArraysToPrivatize](
9848 Function &OutlinedFn) mutable {
9849 assert(OutlinedFn.hasOneUse() &&
9850 "there must be a single user for the outlined function");
9851
9852 CallInst *StaleCI = cast<CallInst>(Val: OutlinedFn.user_back());
9853
9854 // The first argument of StaleCI is always the thread id.
9855 // The next few arguments are the pointers to offloading arrays
9856 // if any. (see OffloadingArraysToPrivatize)
9857 // Finally, all other local values that are live-in into the outlined region
9858 // end up in a structure whose pointer is passed as the last argument. This
9859 // piece of data is passed in the "shared" field of the task structure. So,
9860 // we know we have to pass shareds to the task if the number of arguments is
9861 // greater than OffloadingArraysToPrivatize.size() + 1 The 1 is for the
9862 // thread id. Further, for safety, we assert that the number of arguments of
9863 // StaleCI is exactly OffloadingArraysToPrivatize.size() + 2
9864 const unsigned int NumStaleCIArgs = StaleCI->arg_size();
9865 bool HasShareds = NumStaleCIArgs > OffloadingArraysToPrivatize.size() + 1;
9866 assert((!HasShareds ||
9867 NumStaleCIArgs == (OffloadingArraysToPrivatize.size() + 2)) &&
9868 "Wrong number of arguments for StaleCI when shareds are present");
9869 int SharedArgOperandNo =
9870 HasShareds ? OffloadingArraysToPrivatize.size() + 1 : 0;
9871
9872 StructType *TaskWithPrivatesTy =
9873 createTaskWithPrivatesTy(OMPIRBuilder&: *this, OffloadingArraysToPrivatize);
9874 StructType *PrivatesTy = nullptr;
9875
9876 if (!OffloadingArraysToPrivatize.empty())
9877 PrivatesTy =
9878 static_cast<StructType *>(TaskWithPrivatesTy->getElementType(N: 1));
9879
9880 Function *ProxyFn = emitTargetTaskProxyFunction(
9881 OMPBuilder&: *this, Builder, StaleCI, PrivatesTy, TaskWithPrivatesTy,
9882 NumOffloadingArrays: OffloadingArraysToPrivatize.size(), SharedArgsOperandNo: SharedArgOperandNo);
9883
9884 LLVM_DEBUG(dbgs() << "Proxy task entry function created: " << *ProxyFn
9885 << "\n");
9886
9887 Builder.SetInsertPoint(StaleCI);
9888
9889 // Gather the arguments for emitting the runtime call.
9890 uint32_t SrcLocStrSize;
9891 Constant *SrcLocStr =
9892 getOrCreateSrcLocStr(Loc: LocationDescription(Builder), SrcLocStrSize);
9893 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
9894
9895 // @__kmpc_omp_task_alloc or @__kmpc_omp_target_task_alloc
9896 //
9897 // If `HasNoWait == true`, we call @__kmpc_omp_target_task_alloc to provide
9898 // the DeviceID to the deferred task and also since
9899 // @__kmpc_omp_target_task_alloc creates an untied/async task.
9900 Function *TaskAllocFn =
9901 !NeedsTargetTask
9902 ? getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_alloc)
9903 : getOrCreateRuntimeFunctionPtr(
9904 FnID: OMPRTL___kmpc_omp_target_task_alloc);
9905
9906 // Arguments - `loc_ref` (Ident) and `gtid` (ThreadID)
9907 // call.
9908 Value *ThreadID = getOrCreateThreadID(Ident);
9909
9910 // Argument - `sizeof_kmp_task_t` (TaskSize)
9911 // Tasksize refers to the size in bytes of kmp_task_t data structure
9912 // plus any other data to be passed to the target task, if any, which
9913 // is packed into a struct. kmp_task_t and the struct so created are
9914 // packed into a wrapper struct whose type is TaskWithPrivatesTy.
9915 Value *TaskSize = Builder.getInt64(
9916 C: M.getDataLayout().getTypeStoreSize(Ty: TaskWithPrivatesTy));
9917
9918 // Argument - `sizeof_shareds` (SharedsSize)
9919 // SharedsSize refers to the shareds array size in the kmp_task_t data
9920 // structure.
9921 Value *SharedsSize = Builder.getInt64(C: 0);
9922 if (HasShareds) {
9923 auto *ArgStructAlloca =
9924 dyn_cast<AllocaInst>(Val: StaleCI->getArgOperand(i: SharedArgOperandNo));
9925 assert(ArgStructAlloca &&
9926 "Unable to find the alloca instruction corresponding to arguments "
9927 "for extracted function");
9928 std::optional<TypeSize> ArgAllocSize =
9929 ArgStructAlloca->getAllocationSize(DL: M.getDataLayout());
9930 assert(ArgAllocSize &&
9931 "Unable to determine size of arguments for extracted function");
9932 SharedsSize = Builder.getInt64(C: ArgAllocSize->getFixedValue());
9933 }
9934
9935 // Argument - `flags`
9936 // Task is tied iff (Flags & 1) == 1.
9937 // Task is untied iff (Flags & 1) == 0.
9938 // Task is final iff (Flags & 2) == 2.
9939 // Task is not final iff (Flags & 2) == 0.
9940 // A target task is not final and is untied.
9941 Value *Flags = Builder.getInt32(C: 0);
9942
9943 // Emit the @__kmpc_omp_task_alloc runtime call
9944 // The runtime call returns a pointer to an area where the task captured
9945 // variables must be copied before the task is run (TaskData)
9946 CallInst *TaskData = nullptr;
9947
9948 SmallVector<llvm::Value *> TaskAllocArgs = {
9949 /*loc_ref=*/Ident, /*gtid=*/ThreadID,
9950 /*flags=*/Flags,
9951 /*sizeof_task=*/TaskSize, /*sizeof_shared=*/SharedsSize,
9952 /*task_func=*/ProxyFn};
9953
9954 if (NeedsTargetTask) {
9955 assert(DeviceID && "Expected non-empty device ID.");
9956 TaskAllocArgs.push_back(Elt: DeviceID);
9957 }
9958
9959 TaskData = createRuntimeFunctionCall(Callee: TaskAllocFn, Args: TaskAllocArgs);
9960
9961 Align Alignment = TaskData->getPointerAlignment(DL: M.getDataLayout());
9962 if (HasShareds) {
9963 Value *Shareds = StaleCI->getArgOperand(i: SharedArgOperandNo);
9964 Value *TaskShareds = loadSharedDataFromTaskDescriptor(
9965 OMPIRBuilder&: *this, Builder, TaskWithPrivates: TaskData, TaskWithPrivatesTy);
9966 Builder.CreateMemCpy(Dst: TaskShareds, DstAlign: Alignment, Src: Shareds, SrcAlign: Alignment,
9967 Size: SharedsSize);
9968 }
9969 if (!OffloadingArraysToPrivatize.empty()) {
9970 Value *Privates =
9971 Builder.CreateStructGEP(Ty: TaskWithPrivatesTy, Ptr: TaskData, Idx: 1);
9972 for (unsigned int i = 0; i < OffloadingArraysToPrivatize.size(); ++i) {
9973 Value *PtrToPrivatize = OffloadingArraysToPrivatize[i];
9974 [[maybe_unused]] Type *ArrayType =
9975 getOffloadingArrayType(V: PtrToPrivatize);
9976 assert(ArrayType && "ArrayType cannot be nullptr");
9977
9978 Type *ElementType = PrivatesTy->getElementType(N: i);
9979 assert(ElementType == ArrayType &&
9980 "ElementType should match ArrayType");
9981 (void)ArrayType;
9982
9983 Value *Dst = Builder.CreateStructGEP(Ty: PrivatesTy, Ptr: Privates, Idx: i);
9984 Builder.CreateMemCpy(
9985 Dst, DstAlign: Alignment, Src: PtrToPrivatize, SrcAlign: Alignment,
9986 Size: Builder.getInt64(C: M.getDataLayout().getTypeStoreSize(Ty: ElementType)));
9987 }
9988 }
9989
9990 Value *DepArray = nullptr;
9991 Value *NumDeps = nullptr;
9992 if (Dependencies.DepArray) {
9993 DepArray = Dependencies.DepArray;
9994 NumDeps = Dependencies.NumDeps;
9995 } else if (!Dependencies.Deps.empty()) {
9996 DepArray = emitTaskDependencies(OMPBuilder&: *this, Dependencies: Dependencies.Deps);
9997 NumDeps = Builder.getInt32(C: Dependencies.Deps.size());
9998 }
9999
10000 // ---------------------------------------------------------------
10001 // V5.2 13.8 target construct
10002 // If the nowait clause is present, execution of the target task
10003 // may be deferred. If the nowait clause is not present, the target task is
10004 // an included task.
10005 // ---------------------------------------------------------------
10006 // The above means that the lack of a nowait on the target construct
10007 // translates to '#pragma omp task if(0)'
10008 if (!NeedsTargetTask) {
10009 if (DepArray) {
10010 Function *TaskWaitFn =
10011 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_wait_deps);
10012 createRuntimeFunctionCall(
10013 Callee: TaskWaitFn,
10014 Args: {/*loc_ref=*/Ident, /*gtid=*/ThreadID,
10015 /*ndeps=*/NumDeps,
10016 /*dep_list=*/DepArray,
10017 /*ndeps_noalias=*/ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0),
10018 /*noalias_dep_list=*/
10019 ConstantPointerNull::get(T: PointerType::getUnqual(C&: M.getContext()))});
10020 }
10021 // Included task.
10022 Function *TaskBeginFn =
10023 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_begin_if0);
10024 Function *TaskCompleteFn =
10025 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_complete_if0);
10026 createRuntimeFunctionCall(Callee: TaskBeginFn, Args: {Ident, ThreadID, TaskData});
10027 CallInst *CI = createRuntimeFunctionCall(Callee: ProxyFn, Args: {ThreadID, TaskData});
10028 CI->setDebugLoc(StaleCI->getDebugLoc());
10029 createRuntimeFunctionCall(Callee: TaskCompleteFn, Args: {Ident, ThreadID, TaskData});
10030 } else if (DepArray) {
10031 // HasNoWait - meaning the task may be deferred. Call
10032 // __kmpc_omp_task_with_deps if there are dependencies,
10033 // else call __kmpc_omp_task
10034 Function *TaskFn =
10035 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_with_deps);
10036 createRuntimeFunctionCall(
10037 Callee: TaskFn,
10038 Args: {Ident, ThreadID, TaskData, NumDeps, DepArray,
10039 ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0),
10040 ConstantPointerNull::get(T: PointerType::getUnqual(C&: M.getContext()))});
10041 } else {
10042 // Emit the @__kmpc_omp_task runtime call to spawn the task
10043 Function *TaskFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task);
10044 createRuntimeFunctionCall(Callee: TaskFn, Args: {Ident, ThreadID, TaskData});
10045 }
10046
10047 Builder.ClearInsertionPoint();
10048 StaleCI->eraseFromParent();
10049 for (Instruction *I : llvm::reverse(C&: ToBeDeleted))
10050 I->eraseFromParent();
10051 };
10052 addOutlineInfo(OI: std::move(OI));
10053
10054 LLVM_DEBUG(dbgs() << "Insert block after emitKernelLaunch = \n"
10055 << *(Builder.GetInsertBlock()) << "\n");
10056 LLVM_DEBUG(dbgs() << "Module after emitKernelLaunch = \n"
10057 << *(Builder.GetInsertBlock()->getParent()->getParent())
10058 << "\n");
10059 return Builder.saveIP();
10060}
10061
10062Error OpenMPIRBuilder::emitOffloadingArraysAndArgs(
10063 InsertPointTy AllocaIP, InsertPointTy CodeGenIP, TargetDataInfo &Info,
10064 TargetDataRTArgs &RTArgs, MapInfosTy &CombinedInfo,
10065 CustomMapperCallbackTy CustomMapperCB, bool IsNonContiguous,
10066 bool ForEndCall, function_ref<void(unsigned int, Value *)> DeviceAddrCB) {
10067 if (Error Err =
10068 emitOffloadingArrays(AllocaIP, CodeGenIP, CombinedInfo, Info,
10069 CustomMapperCB, IsNonContiguous, DeviceAddrCB))
10070 return Err;
10071 emitOffloadingArraysArgument(Builder, RTArgs, Info, ForEndCall);
10072 return Error::success();
10073}
10074
10075static void emitTargetCall(
10076 OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder,
10077 OpenMPIRBuilder::InsertPointTy AllocaIP,
10078 ArrayRef<BasicBlock *> DeallocBlocks, OpenMPIRBuilder::TargetDataInfo &Info,
10079 const OpenMPIRBuilder::TargetKernelDefaultAttrs &DefaultAttrs,
10080 const OpenMPIRBuilder::TargetKernelRuntimeAttrs &RuntimeAttrs,
10081 Value *IfCond, Function *OutlinedFn, Constant *OutlinedFnID,
10082 SmallVectorImpl<Value *> &Args,
10083 OpenMPIRBuilder::GenMapInfoCallbackTy GenMapInfoCB,
10084 OpenMPIRBuilder::CustomMapperCallbackTy CustomMapperCB,
10085 const OpenMPIRBuilder::DependenciesInfo &Dependencies, bool HasNoWait,
10086 Value *DynCGroupMem, OMPDynGroupprivateFallbackType DynCGroupMemFallback) {
10087 // Generate a function call to the host fallback implementation of the target
10088 // region. This is called by the host when no offload entry was generated for
10089 // the target region and when the offloading call fails at runtime.
10090 auto &&EmitTargetCallFallbackCB = [&](OpenMPIRBuilder::InsertPointTy IP)
10091 -> OpenMPIRBuilder::InsertPointOrErrorTy {
10092 Builder.restoreIP(IP);
10093 // Ensure the host fallback has the same dyn_ptr ABI as the device.
10094 SmallVector<Value *> FallbackArgs(Args.begin(), Args.end());
10095 FallbackArgs.push_back(
10096 Elt: Constant::getNullValue(Ty: PointerType::getUnqual(C&: Builder.getContext())));
10097 OMPBuilder.createRuntimeFunctionCall(Callee: OutlinedFn, Args: FallbackArgs);
10098 return Builder.saveIP();
10099 };
10100
10101 bool HasDependencies = !Dependencies.empty();
10102 bool RequiresOuterTargetTask = HasNoWait || HasDependencies;
10103
10104 OpenMPIRBuilder::TargetKernelArgs KArgs;
10105
10106 auto TaskBodyCB =
10107 [&](Value *DeviceID, Value *RTLoc,
10108 IRBuilderBase::InsertPoint TargetTaskAllocaIP) -> Error {
10109 // Assume no error was returned because EmitTargetCallFallbackCB doesn't
10110 // produce any.
10111 llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(ValOrErr: [&]() {
10112 // emitKernelLaunch makes the necessary runtime call to offload the
10113 // kernel. We then outline all that code into a separate function
10114 // ('kernel_launch_function' in the pseudo code above). This function is
10115 // then called by the target task proxy function (see
10116 // '@.omp_target_task_proxy_func' in the pseudo code above)
10117 // "@.omp_target_task_proxy_func' is generated by
10118 // emitTargetTaskProxyFunction.
10119 if (OutlinedFnID && DeviceID)
10120 return OMPBuilder.emitKernelLaunch(Loc: Builder, OutlinedFnID,
10121 EmitTargetCallFallbackCB, Args&: KArgs,
10122 DeviceID, RTLoc, AllocaIP: TargetTaskAllocaIP);
10123
10124 // We only need to do the outlining if `DeviceID` is set to avoid calling
10125 // `emitKernelLaunch` if we want to code-gen for the host; e.g. if we are
10126 // generating the `else` branch of an `if` clause.
10127 //
10128 // When OutlinedFnID is set to nullptr, then it's not an offloading call.
10129 // In this case, we execute the host implementation directly.
10130 return EmitTargetCallFallbackCB(OMPBuilder.Builder.saveIP());
10131 }());
10132
10133 OMPBuilder.Builder.restoreIP(IP: AfterIP);
10134 return Error::success();
10135 };
10136
10137 auto &&EmitTargetCallElse =
10138 [&](OpenMPIRBuilder::InsertPointTy AllocaIP,
10139 OpenMPIRBuilder::InsertPointTy CodeGenIP,
10140 ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
10141 // Assume no error was returned because EmitTargetCallFallbackCB doesn't
10142 // produce any.
10143 OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(ValOrErr: [&]() {
10144 if (RequiresOuterTargetTask) {
10145 // Arguments that are intended to be directly forwarded to an
10146 // emitKernelLaunch call are pased as nullptr, since
10147 // OutlinedFnID=nullptr results in that call not being done.
10148 OpenMPIRBuilder::TargetDataRTArgs EmptyRTArgs;
10149 return OMPBuilder.emitTargetTask(TaskBodyCB, /*DeviceID=*/nullptr,
10150 /*RTLoc=*/nullptr, AllocaIP,
10151 Dependencies, RTArgs: EmptyRTArgs, HasNoWait);
10152 }
10153 return EmitTargetCallFallbackCB(Builder.saveIP());
10154 }());
10155
10156 Builder.restoreIP(IP: AfterIP);
10157 return Error::success();
10158 };
10159
10160 auto &&EmitTargetCallThen =
10161 [&](OpenMPIRBuilder::InsertPointTy AllocaIP,
10162 OpenMPIRBuilder::InsertPointTy CodeGenIP,
10163 ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
10164 Info.HasNoWait = HasNoWait;
10165 OpenMPIRBuilder::MapInfosTy &MapInfo = GenMapInfoCB(Builder.saveIP());
10166
10167 OpenMPIRBuilder::TargetDataRTArgs RTArgs;
10168 if (Error Err = OMPBuilder.emitOffloadingArraysAndArgs(
10169 AllocaIP, CodeGenIP: Builder.saveIP(), Info, RTArgs, CombinedInfo&: MapInfo, CustomMapperCB,
10170 /*IsNonContiguous=*/true,
10171 /*ForEndCall=*/false))
10172 return Err;
10173
10174 SmallVector<Value *, 3> NumTeamsC;
10175 for (auto [DefaultVal, RuntimeVal] :
10176 zip_equal(t: DefaultAttrs.MaxTeams, u: RuntimeAttrs.MaxTeams))
10177 NumTeamsC.push_back(Elt: RuntimeVal ? RuntimeVal
10178 : Builder.getInt32(C: DefaultVal));
10179
10180 // Calculate number of threads: 0 if no clauses specified, otherwise it is
10181 // the minimum between optional THREAD_LIMIT and NUM_THREADS clauses.
10182 auto InitMaxThreadsClause = [&Builder](Value *Clause) {
10183 if (Clause)
10184 Clause = Builder.CreateIntCast(V: Clause, DestTy: Builder.getInt32Ty(),
10185 /*isSigned=*/false);
10186 return Clause;
10187 };
10188 auto CombineMaxThreadsClauses = [&Builder](Value *Clause, Value *&Result) {
10189 if (Clause)
10190 Result =
10191 Result ? Builder.CreateSelect(C: Builder.CreateICmpULT(LHS: Result, RHS: Clause),
10192 True: Result, False: Clause)
10193 : Clause;
10194 };
10195
10196 // If a multi-dimensional THREAD_LIMIT is set, it is the OMPX_BARE case, so
10197 // the NUM_THREADS clause is overriden by THREAD_LIMIT.
10198 SmallVector<Value *, 3> NumThreadsC;
10199 Value *MaxThreadsClause =
10200 RuntimeAttrs.TeamsThreadLimit.size() == 1
10201 ? InitMaxThreadsClause(RuntimeAttrs.MaxThreads.front())
10202 : nullptr;
10203
10204 for (auto [TeamsVal, TargetVal] : zip_equal(
10205 t: RuntimeAttrs.TeamsThreadLimit, u: RuntimeAttrs.TargetThreadLimit)) {
10206 Value *TeamsThreadLimitClause = InitMaxThreadsClause(TeamsVal);
10207 Value *NumThreads = InitMaxThreadsClause(TargetVal);
10208
10209 CombineMaxThreadsClauses(TeamsThreadLimitClause, NumThreads);
10210 CombineMaxThreadsClauses(MaxThreadsClause, NumThreads);
10211
10212 NumThreadsC.push_back(Elt: NumThreads ? NumThreads : Builder.getInt32(C: 0));
10213 }
10214
10215 unsigned NumTargetItems = Info.NumberOfPtrs;
10216 uint32_t SrcLocStrSize;
10217 Constant *SrcLocStr = OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
10218 Value *RTLoc = OMPBuilder.getOrCreateIdent(SrcLocStr, SrcLocStrSize,
10219 LocFlags: llvm::omp::IdentFlag(0), Reserve2Flags: 0);
10220
10221 Value *TripCount = RuntimeAttrs.LoopTripCount
10222 ? Builder.CreateIntCast(V: RuntimeAttrs.LoopTripCount,
10223 DestTy: Builder.getInt64Ty(),
10224 /*isSigned=*/false)
10225 : Builder.getInt64(C: 0);
10226
10227 // Request zero groupprivate bytes by default.
10228 if (!DynCGroupMem)
10229 DynCGroupMem = Builder.getInt32(C: 0);
10230
10231 KArgs = OpenMPIRBuilder::TargetKernelArgs(
10232 NumTargetItems, RTArgs, TripCount, NumTeamsC, NumThreadsC, DynCGroupMem,
10233 HasNoWait, /*StrictBlocks=*/false, /*StrictThreads=*/false,
10234 DynCGroupMemFallback);
10235
10236 // Assume no error was returned because TaskBodyCB and
10237 // EmitTargetCallFallbackCB don't produce any.
10238 OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(ValOrErr: [&]() {
10239 // The presence of certain clauses on the target directive require the
10240 // explicit generation of the target task.
10241 if (RequiresOuterTargetTask)
10242 return OMPBuilder.emitTargetTask(TaskBodyCB, DeviceID: RuntimeAttrs.DeviceID,
10243 RTLoc, AllocaIP, Dependencies,
10244 RTArgs: KArgs.RTArgs, HasNoWait: Info.HasNoWait);
10245
10246 return OMPBuilder.emitKernelLaunch(
10247 Loc: Builder, OutlinedFnID, EmitTargetCallFallbackCB, Args&: KArgs,
10248 DeviceID: RuntimeAttrs.DeviceID, RTLoc, AllocaIP);
10249 }());
10250
10251 Builder.restoreIP(IP: AfterIP);
10252 return Error::success();
10253 };
10254
10255 // If we don't have an ID for the target region, it means an offload entry
10256 // wasn't created. In this case we just run the host fallback directly and
10257 // ignore any potential 'if' clauses.
10258 if (!OutlinedFnID) {
10259 cantFail(Err: EmitTargetCallElse(AllocaIP, Builder.saveIP(), DeallocBlocks));
10260 return;
10261 }
10262
10263 // If there's no 'if' clause, only generate the kernel launch code path.
10264 if (!IfCond) {
10265 cantFail(Err: EmitTargetCallThen(AllocaIP, Builder.saveIP(), DeallocBlocks));
10266 return;
10267 }
10268
10269 cantFail(Err: OMPBuilder.emitIfClause(Cond: IfCond, ThenGen: EmitTargetCallThen,
10270 ElseGen: EmitTargetCallElse, AllocaIP));
10271}
10272
10273OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTarget(
10274 const LocationDescription &Loc, bool IsOffloadEntry, InsertPointTy AllocaIP,
10275 InsertPointTy CodeGenIP, ArrayRef<BasicBlock *> DeallocBlocks,
10276 TargetDataInfo &Info, TargetRegionEntryInfo &EntryInfo,
10277 const TargetKernelDefaultAttrs &DefaultAttrs,
10278 const TargetKernelRuntimeAttrs &RuntimeAttrs, Value *IfCond,
10279 SmallVectorImpl<Value *> &Inputs, GenMapInfoCallbackTy GenMapInfoCB,
10280 OpenMPIRBuilder::TargetBodyGenCallbackTy CBFunc,
10281 OpenMPIRBuilder::TargetGenArgAccessorsCallbackTy ArgAccessorFuncCB,
10282 CustomMapperCallbackTy CustomMapperCB, const DependenciesInfo &Dependencies,
10283 bool HasNowait, Value *DynCGroupMem,
10284 OMPDynGroupprivateFallbackType DynCGroupMemFallback,
10285 DebugLoc OutlinedFnLoc) {
10286
10287 if (!updateToLocation(Loc))
10288 return InsertPointTy();
10289
10290 Builder.restoreIP(IP: CodeGenIP);
10291
10292 Function *OutlinedFn;
10293 Constant *OutlinedFnID = nullptr;
10294 // The target region is outlined into its own function. The LLVM IR for
10295 // the target region itself is generated using the callbacks CBFunc
10296 // and ArgAccessorFuncCB
10297 if (Error Err = emitTargetOutlinedFunction(
10298 OMPBuilder&: *this, Builder, IsOffloadEntry, EntryInfo, DefaultAttrs, OutlinedFn,
10299 OutlinedFnID, Inputs, CBFunc, ArgAccessorFuncCB, OutlinedFnLoc))
10300 return Err;
10301
10302 // If we are not on the target device, then we need to generate code
10303 // to make a remote call (offload) to the previously outlined function
10304 // that represents the target region. Do that now.
10305 if (!Config.isTargetDevice())
10306 emitTargetCall(OMPBuilder&: *this, Builder, AllocaIP, DeallocBlocks, Info, DefaultAttrs,
10307 RuntimeAttrs, IfCond, OutlinedFn, OutlinedFnID, Args&: Inputs,
10308 GenMapInfoCB, CustomMapperCB, Dependencies, HasNoWait: HasNowait,
10309 DynCGroupMem, DynCGroupMemFallback);
10310 return Builder.saveIP();
10311}
10312
10313std::string OpenMPIRBuilder::getNameWithSeparators(ArrayRef<StringRef> Parts,
10314 StringRef FirstSeparator,
10315 StringRef Separator) {
10316 SmallString<128> Buffer;
10317 llvm::raw_svector_ostream OS(Buffer);
10318 StringRef Sep = FirstSeparator;
10319 for (StringRef Part : Parts) {
10320 OS << Sep << Part;
10321 Sep = Separator;
10322 }
10323 return OS.str().str();
10324}
10325
10326std::string
10327OpenMPIRBuilder::createPlatformSpecificName(ArrayRef<StringRef> Parts) const {
10328 return OpenMPIRBuilder::getNameWithSeparators(Parts, FirstSeparator: Config.firstSeparator(),
10329 Separator: Config.separator());
10330}
10331
10332GlobalVariable *OpenMPIRBuilder::getOrCreateInternalVariable(
10333 Type *Ty, const StringRef &Name, std::optional<unsigned> AddressSpace) {
10334 auto &Elem = *InternalVars.try_emplace(Key: Name, Args: nullptr).first;
10335 if (Elem.second) {
10336 assert(Elem.second->getValueType() == Ty &&
10337 "OMP internal variable has different type than requested");
10338 } else {
10339 // TODO: investigate the appropriate linkage type used for the global
10340 // variable for possibly changing that to internal or private, or maybe
10341 // create different versions of the function for different OMP internal
10342 // variables.
10343 const DataLayout &DL = M.getDataLayout();
10344 // TODO: Investigate why AMDGPU expects AS 0 for globals even though the
10345 // default global AS is 1.
10346 // See double-target-call-with-declare-target.f90 and
10347 // declare-target-vars-in-target-region.f90 libomptarget
10348 // tests.
10349 unsigned AddressSpaceVal = AddressSpace ? *AddressSpace
10350 : M.getTargetTriple().isAMDGPU()
10351 ? 0
10352 : DL.getDefaultGlobalsAddressSpace();
10353 auto Linkage = this->M.getTargetTriple().isWasm()
10354 ? GlobalValue::InternalLinkage
10355 : GlobalValue::CommonLinkage;
10356 auto *GV = new GlobalVariable(M, Ty, /*IsConstant=*/false, Linkage,
10357 Constant::getNullValue(Ty), Elem.first(),
10358 /*InsertBefore=*/nullptr,
10359 GlobalValue::NotThreadLocal, AddressSpaceVal);
10360 const llvm::Align TypeAlign = DL.getABITypeAlign(Ty);
10361 const llvm::Align PtrAlign = DL.getPointerABIAlignment(AS: AddressSpaceVal);
10362 GV->setAlignment(std::max(a: TypeAlign, b: PtrAlign));
10363 Elem.second = GV;
10364 }
10365
10366 return Elem.second;
10367}
10368
10369Value *OpenMPIRBuilder::getOMPCriticalRegionLock(StringRef CriticalName) {
10370 std::string Prefix = Twine("gomp_critical_user_", CriticalName).str();
10371 std::string Name = getNameWithSeparators(Parts: {Prefix, "var"}, FirstSeparator: ".", Separator: ".");
10372 return getOrCreateInternalVariable(Ty: KmpCriticalNameTy, Name);
10373}
10374
10375Value *OpenMPIRBuilder::getSizeInBytes(Value *BasePtr) {
10376 LLVMContext &Ctx = Builder.getContext();
10377 Value *Null =
10378 Constant::getNullValue(Ty: PointerType::getUnqual(C&: BasePtr->getContext()));
10379 Value *SizeGep =
10380 Builder.CreateGEP(Ty: BasePtr->getType(), Ptr: Null, IdxList: Builder.getInt32(C: 1));
10381 Value *SizePtrToInt = Builder.CreatePtrToInt(V: SizeGep, DestTy: Type::getInt64Ty(C&: Ctx));
10382 return SizePtrToInt;
10383}
10384
10385GlobalVariable *
10386OpenMPIRBuilder::createOffloadMaptypes(SmallVectorImpl<uint64_t> &Mappings,
10387 std::string VarName) {
10388 llvm::Constant *MaptypesArrayInit =
10389 llvm::ConstantDataArray::get(Context&: M.getContext(), Elts&: Mappings);
10390 auto *MaptypesArrayGlobal = new llvm::GlobalVariable(
10391 M, MaptypesArrayInit->getType(),
10392 /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage, MaptypesArrayInit,
10393 VarName);
10394 MaptypesArrayGlobal->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
10395 return MaptypesArrayGlobal;
10396}
10397
10398void OpenMPIRBuilder::createMapperAllocas(const LocationDescription &Loc,
10399 InsertPointTy AllocaIP,
10400 unsigned NumOperands,
10401 struct MapperAllocas &MapperAllocas) {
10402 if (!updateToLocation(Loc))
10403 return;
10404
10405 auto *ArrI8PtrTy = ArrayType::get(ElementType: Int8Ptr, NumElements: NumOperands);
10406 auto *ArrI64Ty = ArrayType::get(ElementType: Int64, NumElements: NumOperands);
10407 Builder.restoreIP(IP: AllocaIP);
10408 AllocaInst *ArgsBase = Builder.CreateAlloca(
10409 Ty: ArrI8PtrTy, /* ArraySize = */ nullptr, Name: ".offload_baseptrs");
10410 AllocaInst *Args = Builder.CreateAlloca(Ty: ArrI8PtrTy, /* ArraySize = */ nullptr,
10411 Name: ".offload_ptrs");
10412 AllocaInst *ArgSizes = Builder.CreateAlloca(
10413 Ty: ArrI64Ty, /* ArraySize = */ nullptr, Name: ".offload_sizes");
10414 updateToLocation(Loc);
10415 MapperAllocas.ArgsBase = ArgsBase;
10416 MapperAllocas.Args = Args;
10417 MapperAllocas.ArgSizes = ArgSizes;
10418}
10419
10420void OpenMPIRBuilder::emitMapperCall(const LocationDescription &Loc,
10421 Function *MapperFunc, Value *SrcLocInfo,
10422 Value *MaptypesArg, Value *MapnamesArg,
10423 struct MapperAllocas &MapperAllocas,
10424 int64_t DeviceID, unsigned NumOperands) {
10425 if (!updateToLocation(Loc))
10426 return;
10427
10428 auto *ArrI8PtrTy = ArrayType::get(ElementType: Int8Ptr, NumElements: NumOperands);
10429 auto *ArrI64Ty = ArrayType::get(ElementType: Int64, NumElements: NumOperands);
10430 Value *ArgsBaseGEP =
10431 Builder.CreateInBoundsGEP(Ty: ArrI8PtrTy, Ptr: MapperAllocas.ArgsBase,
10432 IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: 0)});
10433 Value *ArgsGEP =
10434 Builder.CreateInBoundsGEP(Ty: ArrI8PtrTy, Ptr: MapperAllocas.Args,
10435 IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: 0)});
10436 Value *ArgSizesGEP =
10437 Builder.CreateInBoundsGEP(Ty: ArrI64Ty, Ptr: MapperAllocas.ArgSizes,
10438 IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: 0)});
10439 Value *NullPtr =
10440 Constant::getNullValue(Ty: PointerType::getUnqual(C&: Int8Ptr->getContext()));
10441 createRuntimeFunctionCall(Callee: MapperFunc, Args: {SrcLocInfo, Builder.getInt64(C: DeviceID),
10442 Builder.getInt32(C: NumOperands),
10443 ArgsBaseGEP, ArgsGEP, ArgSizesGEP,
10444 MaptypesArg, MapnamesArg, NullPtr});
10445}
10446
10447void OpenMPIRBuilder::emitOffloadingArraysArgument(IRBuilderBase &Builder,
10448 TargetDataRTArgs &RTArgs,
10449 TargetDataInfo &Info,
10450 bool ForEndCall) {
10451 assert((!ForEndCall || Info.separateBeginEndCalls()) &&
10452 "expected region end call to runtime only when end call is separate");
10453 auto UnqualPtrTy = PointerType::getUnqual(C&: M.getContext());
10454 auto VoidPtrTy = UnqualPtrTy;
10455 auto VoidPtrPtrTy = UnqualPtrTy;
10456 auto Int64Ty = Type::getInt64Ty(C&: M.getContext());
10457 auto Int64PtrTy = UnqualPtrTy;
10458
10459 if (!Info.NumberOfPtrs) {
10460 RTArgs.BasePointersArray = ConstantPointerNull::get(T: VoidPtrPtrTy);
10461 RTArgs.PointersArray = ConstantPointerNull::get(T: VoidPtrPtrTy);
10462 RTArgs.SizesArray = ConstantPointerNull::get(T: Int64PtrTy);
10463 RTArgs.MapTypesArray = ConstantPointerNull::get(T: Int64PtrTy);
10464 RTArgs.MapNamesArray = ConstantPointerNull::get(T: VoidPtrPtrTy);
10465 RTArgs.MappersArray = ConstantPointerNull::get(T: VoidPtrPtrTy);
10466 return;
10467 }
10468
10469 RTArgs.BasePointersArray = Builder.CreateConstInBoundsGEP2_32(
10470 Ty: ArrayType::get(ElementType: VoidPtrTy, NumElements: Info.NumberOfPtrs),
10471 Ptr: Info.RTArgs.BasePointersArray,
10472 /*Idx0=*/0, /*Idx1=*/0);
10473 RTArgs.PointersArray = Builder.CreateConstInBoundsGEP2_32(
10474 Ty: ArrayType::get(ElementType: VoidPtrTy, NumElements: Info.NumberOfPtrs), Ptr: Info.RTArgs.PointersArray,
10475 /*Idx0=*/0,
10476 /*Idx1=*/0);
10477 RTArgs.SizesArray = Builder.CreateConstInBoundsGEP2_32(
10478 Ty: ArrayType::get(ElementType: Int64Ty, NumElements: Info.NumberOfPtrs), Ptr: Info.RTArgs.SizesArray,
10479 /*Idx0=*/0, /*Idx1=*/0);
10480 RTArgs.MapTypesArray = Builder.CreateConstInBoundsGEP2_32(
10481 Ty: ArrayType::get(ElementType: Int64Ty, NumElements: Info.NumberOfPtrs),
10482 Ptr: ForEndCall && Info.RTArgs.MapTypesArrayEnd ? Info.RTArgs.MapTypesArrayEnd
10483 : Info.RTArgs.MapTypesArray,
10484 /*Idx0=*/0,
10485 /*Idx1=*/0);
10486
10487 // Only emit the mapper information arrays if debug information is
10488 // requested.
10489 if (!Info.EmitDebug)
10490 RTArgs.MapNamesArray = ConstantPointerNull::get(T: VoidPtrPtrTy);
10491 else
10492 RTArgs.MapNamesArray = Builder.CreateConstInBoundsGEP2_32(
10493 Ty: ArrayType::get(ElementType: VoidPtrTy, NumElements: Info.NumberOfPtrs), Ptr: Info.RTArgs.MapNamesArray,
10494 /*Idx0=*/0,
10495 /*Idx1=*/0);
10496 // If there is no user-defined mapper, set the mapper array to nullptr to
10497 // avoid an unnecessary data privatization
10498 if (!Info.HasMapper)
10499 RTArgs.MappersArray = ConstantPointerNull::get(T: VoidPtrPtrTy);
10500 else
10501 RTArgs.MappersArray =
10502 Builder.CreatePointerCast(V: Info.RTArgs.MappersArray, DestTy: VoidPtrPtrTy);
10503}
10504
10505void OpenMPIRBuilder::emitNonContiguousDescriptor(InsertPointTy AllocaIP,
10506 InsertPointTy CodeGenIP,
10507 MapInfosTy &CombinedInfo,
10508 TargetDataInfo &Info) {
10509 MapInfosTy::StructNonContiguousInfo &NonContigInfo =
10510 CombinedInfo.NonContigInfo;
10511
10512 // Build an array of struct descriptor_dim and then assign it to
10513 // offload_args.
10514 //
10515 // struct descriptor_dim {
10516 // uint64_t offset;
10517 // uint64_t count;
10518 // uint64_t stride
10519 // };
10520 Type *Int64Ty = Builder.getInt64Ty();
10521 StructType *DimTy = StructType::create(
10522 Context&: M.getContext(), Elements: ArrayRef<Type *>({Int64Ty, Int64Ty, Int64Ty}),
10523 Name: "struct.descriptor_dim");
10524
10525 enum { OffsetFD = 0, CountFD, StrideFD };
10526 // We need two index variable here since the size of "Dims" is the same as
10527 // the size of Components, however, the size of offset, count, and stride is
10528 // equal to the size of base declaration that is non-contiguous.
10529 for (unsigned I = 0, L = 0, E = NonContigInfo.Dims.size(); I < E; ++I) {
10530 // Skip emitting ir if dimension size is 1 since it cannot be
10531 // non-contiguous.
10532 if (NonContigInfo.Dims[I] == 1)
10533 continue;
10534 Builder.restoreIP(IP: AllocaIP);
10535 ArrayType *ArrayTy = ArrayType::get(ElementType: DimTy, NumElements: NonContigInfo.Dims[I]);
10536 AllocaInst *DimsAddr =
10537 Builder.CreateAlloca(Ty: ArrayTy, /* ArraySize = */ nullptr, Name: "dims");
10538 Builder.restoreIP(IP: CodeGenIP);
10539 for (unsigned II = 0, EE = NonContigInfo.Dims[I]; II < EE; ++II) {
10540 unsigned RevIdx = EE - II - 1;
10541 Value *DimsLVal = Builder.CreateInBoundsGEP(
10542 Ty: ArrayTy, Ptr: DimsAddr, IdxList: {Builder.getInt64(C: 0), Builder.getInt64(C: II)});
10543 // Offset
10544 Value *OffsetLVal = Builder.CreateStructGEP(Ty: DimTy, Ptr: DimsLVal, Idx: OffsetFD);
10545 Builder.CreateAlignedStore(
10546 Val: NonContigInfo.Offsets[L][RevIdx], Ptr: OffsetLVal,
10547 Align: M.getDataLayout().getPrefTypeAlign(Ty: OffsetLVal->getType()));
10548 // Count
10549 Value *CountLVal = Builder.CreateStructGEP(Ty: DimTy, Ptr: DimsLVal, Idx: CountFD);
10550 Builder.CreateAlignedStore(
10551 Val: NonContigInfo.Counts[L][RevIdx], Ptr: CountLVal,
10552 Align: M.getDataLayout().getPrefTypeAlign(Ty: CountLVal->getType()));
10553 // Stride
10554 Value *StrideLVal = Builder.CreateStructGEP(Ty: DimTy, Ptr: DimsLVal, Idx: StrideFD);
10555 Builder.CreateAlignedStore(
10556 Val: NonContigInfo.Strides[L][RevIdx], Ptr: StrideLVal,
10557 Align: M.getDataLayout().getPrefTypeAlign(Ty: CountLVal->getType()));
10558 }
10559 // args[I] = &dims
10560 Builder.restoreIP(IP: CodeGenIP);
10561 Value *DAddr = Builder.CreatePointerBitCastOrAddrSpaceCast(
10562 V: DimsAddr, DestTy: Builder.getPtrTy());
10563 Value *P = Builder.CreateConstInBoundsGEP2_32(
10564 Ty: ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: Info.NumberOfPtrs),
10565 Ptr: Info.RTArgs.PointersArray, Idx0: 0, Idx1: I);
10566 Builder.CreateAlignedStore(
10567 Val: DAddr, Ptr: P, Align: M.getDataLayout().getPrefTypeAlign(Ty: Builder.getPtrTy()));
10568 ++L;
10569 }
10570}
10571
10572void OpenMPIRBuilder::emitUDMapperArrayInitOrDel(
10573 Function *MapperFn, Value *MapperHandle, Value *Base, Value *Begin,
10574 Value *Size, Value *MapType, Value *MapName, TypeSize ElementSize,
10575 BasicBlock *ExitBB, bool IsInit) {
10576 StringRef Prefix = IsInit ? ".init" : ".del";
10577
10578 // Evaluate if this is an array section.
10579 BasicBlock *BodyBB = BasicBlock::Create(
10580 Context&: M.getContext(), Name: createPlatformSpecificName(Parts: {"omp.array", Prefix}));
10581 Value *IsArray =
10582 Builder.CreateICmpSGT(LHS: Size, RHS: Builder.getInt64(C: 1), Name: "omp.arrayinit.isarray");
10583 Value *DeleteBit = Builder.CreateAnd(
10584 LHS: MapType,
10585 RHS: Builder.getInt64(
10586 C: static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10587 OpenMPOffloadMappingFlags::OMP_MAP_DELETE)));
10588 Value *DeleteCond;
10589 Value *Cond;
10590 if (IsInit) {
10591 // base != begin?
10592 Value *BaseIsBegin = Builder.CreateICmpNE(LHS: Base, RHS: Begin);
10593 Cond = Builder.CreateOr(LHS: IsArray, RHS: BaseIsBegin);
10594 DeleteCond = Builder.CreateIsNull(
10595 Arg: DeleteBit,
10596 Name: createPlatformSpecificName(Parts: {"omp.array", Prefix, ".delete"}));
10597 } else {
10598 Cond = IsArray;
10599 DeleteCond = Builder.CreateIsNotNull(
10600 Arg: DeleteBit,
10601 Name: createPlatformSpecificName(Parts: {"omp.array", Prefix, ".delete"}));
10602 }
10603 Cond = Builder.CreateAnd(LHS: Cond, RHS: DeleteCond);
10604 Builder.CreateCondBr(Cond, True: BodyBB, False: ExitBB);
10605
10606 emitBlock(BB: BodyBB, CurFn: MapperFn);
10607 // Get the array size by multiplying element size and element number (i.e., \p
10608 // Size).
10609 Value *ArraySize = Builder.CreateNUWMul(LHS: Size, RHS: Builder.getInt64(C: ElementSize));
10610 // Remove OMP_MAP_TO and OMP_MAP_FROM from the map type, so that it achieves
10611 // memory allocation/deletion purpose only.
10612 Value *MapTypeArg = Builder.CreateAnd(
10613 LHS: MapType,
10614 RHS: Builder.getInt64(
10615 C: ~static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10616 OpenMPOffloadMappingFlags::OMP_MAP_TO |
10617 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
10618 MapTypeArg = Builder.CreateOr(
10619 LHS: MapTypeArg,
10620 RHS: Builder.getInt64(
10621 C: static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10622 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT)));
10623
10624 // Call the runtime API __tgt_push_mapper_component to fill up the runtime
10625 // data structure.
10626 Value *OffloadingArgs[] = {MapperHandle, Base, Begin,
10627 ArraySize, MapTypeArg, MapName};
10628 createRuntimeFunctionCall(
10629 Callee: getOrCreateRuntimeFunction(M, FnID: OMPRTL___tgt_push_mapper_component),
10630 Args: OffloadingArgs);
10631}
10632
10633Expected<Function *> OpenMPIRBuilder::emitUserDefinedMapper(
10634 function_ref<MapInfosOrErrorTy(InsertPointTy CodeGenIP, llvm::Value *PtrPHI,
10635 llvm::Value *BeginArg)>
10636 GenMapInfoCB,
10637 Type *ElemTy, StringRef FuncName, CustomMapperCallbackTy CustomMapperCB,
10638 bool PreserveMemberOfFlags, bool PropagatePresentToPointee) {
10639 SmallVector<Type *> Params;
10640 Params.emplace_back(Args: Builder.getPtrTy());
10641 Params.emplace_back(Args: Builder.getPtrTy());
10642 Params.emplace_back(Args: Builder.getPtrTy());
10643 Params.emplace_back(Args: Builder.getInt64Ty());
10644 Params.emplace_back(Args: Builder.getInt64Ty());
10645 Params.emplace_back(Args: Builder.getPtrTy());
10646
10647 auto *FnTy =
10648 FunctionType::get(Result: Builder.getVoidTy(), Params, /* IsVarArg */ isVarArg: false);
10649
10650 SmallString<64> TyStr;
10651 raw_svector_ostream Out(TyStr);
10652 Function *MapperFn =
10653 Function::Create(Ty: FnTy, Linkage: GlobalValue::InternalLinkage, N: FuncName, M);
10654 MapperFn->addFnAttr(Kind: Attribute::NoInline);
10655 MapperFn->addFnAttr(Kind: Attribute::NoUnwind);
10656 MapperFn->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
10657 MapperFn->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
10658 MapperFn->addParamAttr(ArgNo: 2, Kind: Attribute::NoUndef);
10659 MapperFn->addParamAttr(ArgNo: 3, Kind: Attribute::NoUndef);
10660 MapperFn->addParamAttr(ArgNo: 4, Kind: Attribute::NoUndef);
10661 MapperFn->addParamAttr(ArgNo: 5, Kind: Attribute::NoUndef);
10662
10663 // Start the mapper function code generation.
10664 BasicBlock *EntryBB = BasicBlock::Create(Context&: M.getContext(), Name: "entry", Parent: MapperFn);
10665 IRBuilder<>::InsertPointGuard IPG(Builder);
10666 Builder.SetInsertPoint(EntryBB);
10667 Builder.SetCurrentDebugLocation(llvm::DebugLoc());
10668
10669 Value *MapperHandle = MapperFn->getArg(i: 0);
10670 Value *BaseIn = MapperFn->getArg(i: 1);
10671 Value *BeginIn = MapperFn->getArg(i: 2);
10672 Value *Size = MapperFn->getArg(i: 3);
10673 Value *MapType = MapperFn->getArg(i: 4);
10674 Value *MapName = MapperFn->getArg(i: 5);
10675
10676 // Compute the starting and end addresses of array elements.
10677 // Prepare common arguments for array initiation and deletion.
10678 // Convert the size in bytes into the number of array elements.
10679 TypeSize ElementSize = M.getDataLayout().getTypeStoreSize(Ty: ElemTy);
10680 Size = Builder.CreateExactUDiv(LHS: Size, RHS: Builder.getInt64(C: ElementSize));
10681 Value *PtrBegin = BeginIn;
10682 Value *PtrEnd = Builder.CreateGEP(Ty: ElemTy, Ptr: PtrBegin, IdxList: Size);
10683
10684 // Emit array initiation if this is an array section and \p MapType indicates
10685 // that memory allocation is required.
10686 BasicBlock *HeadBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp.arraymap.head");
10687 emitUDMapperArrayInitOrDel(MapperFn, MapperHandle, Base: BaseIn, Begin: BeginIn, Size,
10688 MapType, MapName, ElementSize, ExitBB: HeadBB,
10689 /*IsInit=*/true);
10690
10691 // Emit a for loop to iterate through SizeArg of elements and map all of them.
10692
10693 // Emit the loop header block.
10694 emitBlock(BB: HeadBB, CurFn: MapperFn);
10695 BasicBlock *BodyBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp.arraymap.body");
10696 BasicBlock *DoneBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp.done");
10697 // Evaluate whether the initial condition is satisfied.
10698 Value *IsEmpty =
10699 Builder.CreateICmpEQ(LHS: PtrBegin, RHS: PtrEnd, Name: "omp.arraymap.isempty");
10700 Builder.CreateCondBr(Cond: IsEmpty, True: DoneBB, False: BodyBB);
10701
10702 // Emit the loop body block.
10703 emitBlock(BB: BodyBB, CurFn: MapperFn);
10704 BasicBlock *LastBB = BodyBB;
10705 PHINode *PtrPHI =
10706 Builder.CreatePHI(Ty: PtrBegin->getType(), NumReservedValues: 2, Name: "omp.arraymap.ptrcurrent");
10707 PtrPHI->addIncoming(V: PtrBegin, BB: HeadBB);
10708
10709 // Get map clause information. Fill up the arrays with all mapped variables.
10710 MapInfosOrErrorTy Info = GenMapInfoCB(Builder.saveIP(), PtrPHI, BeginIn);
10711 if (!Info)
10712 return Info.takeError();
10713
10714 // Call the runtime API __tgt_mapper_num_components to get the number of
10715 // pre-existing components.
10716 Value *OffloadingArgs[] = {MapperHandle};
10717 Value *PreviousSize = createRuntimeFunctionCall(
10718 Callee: getOrCreateRuntimeFunction(M, FnID: OMPRTL___tgt_mapper_num_components),
10719 Args: OffloadingArgs);
10720 Value *ShiftedPreviousSize =
10721 Builder.CreateShl(LHS: PreviousSize, RHS: Builder.getInt64(C: getFlagMemberOffset()));
10722
10723 // Fill up the runtime mapper handle for all components.
10724 for (unsigned I = 0; I < Info->BasePointers.size(); ++I) {
10725 Value *CurBaseArg = Info->BasePointers[I];
10726 Value *CurBeginArg = Info->Pointers[I];
10727 Value *CurSizeArg = Info->Sizes[I];
10728 Value *CurNameArg = Info->Names.size()
10729 ? Info->Names[I]
10730 : Constant::getNullValue(Ty: Builder.getPtrTy());
10731
10732 Value *OriMapType = Builder.getInt64(
10733 C: static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10734 Info->Types[I]));
10735 auto RawType =
10736 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10737 Info->Types[I]);
10738 constexpr uint64_t MemberOfMask =
10739 static_cast<uint64_t>(OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);
10740 constexpr uint64_t AttachBit =
10741 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10742 OpenMPOffloadMappingFlags::OMP_MAP_ATTACH);
10743
10744 // Add MEMBER_OF (ShiftedPreviousSize) to group this sub-map with the
10745 // current array element (N = __tgt_mapper_num_components() at loop body
10746 // start).
10747 //
10748 // Example 1:
10749 // struct S { int x; int *p; };
10750 //
10751 // mapper: #pragma omp declare mapper(id: S s) map(s.x, s.p[0:10])
10752 // use: S arr[2]; ... map(arr)
10753 // entries per element:
10754 //
10755 // &arr[i], &arr[i].x, sizeof(int), MEMBER_OF(N)|TO|FROM
10756 // &arr[i].p[0], &arr[i].p[0], 10*sizeof(int), TO|FROM (*)
10757 // &arr[i].p, &arr[i].p[0], sizeof(int*), ATTACH (**)
10758 //
10759 // Example 2:
10760 // struct S1 { int x; int y; };
10761 // struct S2 { int z; S1 *s1p; };
10762 //
10763 // mapper: #pragma omp declare mapper(S2 s2) map(s2.z, s2.s1p->x,
10764 // s2.s1p->y)
10765 // use: S2 arr[2]; ... map(arr)
10766 // entries per element:
10767 //
10768 // &arr[i], &arr[i].z, sizeof(int), MEMBER_OF(N)|TO|FROM
10769 // &arr[i].s1p[0], &arr[i].s1p->x, sizeof(s1p->x..y), ALLOC (*)
10770 // &arr[i].s1p[0], &arr[i].s1p->x, 4, MEMBER_OF(N+2)|TO|FROM (*)(***)
10771 // &arr[i].s1p[0], &arr[i].s1p->y, 4, MEMBER_OF(N+2)|TO|FROM (*)(***)
10772 // &arr[i].s1p, &arr[i].s1p->x, sizeof(ptr), ATTACH (**)
10773 //
10774 // x/y carry inner MEMBER_OF(2)
10775 // which is shifted by N to become MEMBER_OF(N+2).
10776 //
10777 // HasAttachPtr is set on all of the s1p entries except the ATTACH one:
10778 // the combined ALLOC entry for the s1p->x..y block, and the individual
10779 // x/y entries that are MEMBER_OF that block, all describe storage
10780 // reached through the attach ptr arr[i].s1p.
10781 //
10782 // Entries of the following kinds do NOT receive a new outer MEMBER_OF
10783 // linking them to the parent struct:
10784 //
10785 // * (*) Entries with HasAttachPtr: they represent pointee data that
10786 // occupies a different storage block than the struct being mapped, so
10787 // they are not a member of it. They may still be MEMBER_OF an entry
10788 // within that pointee block, in which case those pre-existing bits are
10789 // shifted -- see (***).
10790 // * (**) ATTACH entries: they are not a member of anything — they just
10791 // link a ptr to its ptee.
10792 // * All entries when PreserveMemberOfFlags is set (the Flang/MLIR path):
10793 // its pre-shaped entries already carry their final MEMBER_OF bits.
10794 // TODO: set HasAttachPtr from Flang for entries whose storage is the
10795 // pointee's (e.g. s%p(0:10)) and drop PreserveMemberOfFlags in favor of
10796 // it.
10797 //
10798 // (***) If such an entry already has its own MEMBER_OF bits (e.g. the
10799 // s1p->x/y entries above), those bits are still shifted by N.
10800 Value *MemberMapType;
10801 if (PreserveMemberOfFlags || (RawType & AttachBit) ||
10802 Info->HasAttachPtr[I]) {
10803 if (RawType & MemberOfMask)
10804 MemberMapType = Builder.CreateNUWAdd(LHS: OriMapType, RHS: ShiftedPreviousSize);
10805 else
10806 MemberMapType = OriMapType;
10807 } else {
10808 MemberMapType = Builder.CreateNUWAdd(LHS: OriMapType, RHS: ShiftedPreviousSize);
10809 }
10810
10811 // Combine the map type inherited from user-defined mapper with that
10812 // specified in the program. According to the OMP_MAP_TO and OMP_MAP_FROM
10813 // bits of the \a MapType, which is the input argument of the mapper
10814 // function, the following code will set the OMP_MAP_TO and OMP_MAP_FROM
10815 // bits of MemberMapType.
10816 // [OpenMP 5.0], 1.2.6. map-type decay.
10817 // | alloc | to | from | tofrom | release | delete
10818 // ----------------------------------------------------------
10819 // alloc | alloc | alloc | alloc | alloc | release | delete
10820 // to | alloc | to | alloc | to | release | delete
10821 // from | alloc | alloc | from | from | release | delete
10822 // tofrom | alloc | to | from | tofrom | release | delete
10823 Value *LeftToFrom = Builder.CreateAnd(
10824 LHS: MapType,
10825 RHS: Builder.getInt64(
10826 C: static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10827 OpenMPOffloadMappingFlags::OMP_MAP_TO |
10828 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
10829 BasicBlock *AllocBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp.type.alloc");
10830 BasicBlock *AllocElseBB =
10831 BasicBlock::Create(Context&: M.getContext(), Name: "omp.type.alloc.else");
10832 BasicBlock *ToBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp.type.to");
10833 BasicBlock *ToElseBB =
10834 BasicBlock::Create(Context&: M.getContext(), Name: "omp.type.to.else");
10835 BasicBlock *FromBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp.type.from");
10836 BasicBlock *EndBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp.type.end");
10837 Value *IsAlloc = Builder.CreateIsNull(Arg: LeftToFrom);
10838 Builder.CreateCondBr(Cond: IsAlloc, True: AllocBB, False: AllocElseBB);
10839 // In case of alloc, clear OMP_MAP_TO and OMP_MAP_FROM.
10840 emitBlock(BB: AllocBB, CurFn: MapperFn);
10841 Value *AllocMapType = Builder.CreateAnd(
10842 LHS: MemberMapType,
10843 RHS: Builder.getInt64(
10844 C: ~static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10845 OpenMPOffloadMappingFlags::OMP_MAP_TO |
10846 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
10847 Builder.CreateBr(Dest: EndBB);
10848 emitBlock(BB: AllocElseBB, CurFn: MapperFn);
10849 Value *IsTo = Builder.CreateICmpEQ(
10850 LHS: LeftToFrom,
10851 RHS: Builder.getInt64(
10852 C: static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10853 OpenMPOffloadMappingFlags::OMP_MAP_TO)));
10854 Builder.CreateCondBr(Cond: IsTo, True: ToBB, False: ToElseBB);
10855 // In case of to, clear OMP_MAP_FROM.
10856 emitBlock(BB: ToBB, CurFn: MapperFn);
10857 Value *ToMapType = Builder.CreateAnd(
10858 LHS: MemberMapType,
10859 RHS: Builder.getInt64(
10860 C: ~static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10861 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
10862 Builder.CreateBr(Dest: EndBB);
10863 emitBlock(BB: ToElseBB, CurFn: MapperFn);
10864 Value *IsFrom = Builder.CreateICmpEQ(
10865 LHS: LeftToFrom,
10866 RHS: Builder.getInt64(
10867 C: static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10868 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
10869 Builder.CreateCondBr(Cond: IsFrom, True: FromBB, False: EndBB);
10870 // In case of from, clear OMP_MAP_TO.
10871 emitBlock(BB: FromBB, CurFn: MapperFn);
10872 Value *FromMapType = Builder.CreateAnd(
10873 LHS: MemberMapType,
10874 RHS: Builder.getInt64(
10875 C: ~static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10876 OpenMPOffloadMappingFlags::OMP_MAP_TO)));
10877 // In case of tofrom, do nothing.
10878 emitBlock(BB: EndBB, CurFn: MapperFn);
10879 LastBB = EndBB;
10880 PHINode *CurMapType =
10881 Builder.CreatePHI(Ty: Builder.getInt64Ty(), NumReservedValues: 4, Name: "omp.maptype");
10882 CurMapType->addIncoming(V: AllocMapType, BB: AllocBB);
10883 CurMapType->addIncoming(V: ToMapType, BB: ToBB);
10884 CurMapType->addIncoming(V: FromMapType, BB: FromBB);
10885 CurMapType->addIncoming(V: MemberMapType, BB: ToElseBB);
10886
10887 // Propagate map-type-modifying bits from the outer map clause to each map
10888 // inserted by the mapper.
10889 //
10890 // OpenMP 6.0:281:34: The effect of the mapper modifier is to remove the
10891 // list item from the map clause and to apply the clauses specified in the
10892 // declared mapper to the construct on which the map clause appears...
10893 // If any modifier with the map-type-modifying property appears in the map
10894 // clause then the effect is as if that modifier appears in each map clause
10895 // specified in the declared mapper.
10896 //
10897 // Map-type-modifying bits: ALWAYS, DELETE, CLOSE, PRESENT.
10898 //
10899 // ALWAYS/DELETE/CLOSE are propagated to every (non-ATTACH) entry.
10900 //
10901 // PRESENT is propagated only to entries that have an attach ptr
10902 // (HasAttachPtr): the pointee data, which occupies a different storage
10903 // block than the struct being mapped and so is not covered by the
10904 // present-check on the struct's own storage. A present modifier on the
10905 // outer clause must still require that pointee to be present on the device.
10906 //
10907 // This is gated on \p PropagatePresentToPointee (set by callers only for
10908 // OpenMP >= 6.0). Before 6.0 the present modifier is treated as not
10909 // applying to the pointee: the spec committee confirmed the divergence
10910 // between the present "motion" modifier (to/from) and the present map-type
10911 // modifier (map) was unintentional, to be fixed as an OpenMP 6.0 erratum,
10912 // so for 5.2 present is ignored for the pointee for both map and to/from.
10913 //
10914 // TODO: PRESENT should also be propagated to the struct's own members
10915 // (e.g. the s.x, s.y of map(present, mapper(id): s)) so that an absent
10916 // member triggers the present-check. We cannot do that yet: while pointer
10917 // members are mapped with PTR_AND_OBJ, a single combined entry allocates
10918 // the whole struct (including the pointer's storage), so propagating
10919 // PRESENT to it would wrongly require the pointer's pointee to be present.
10920 // Enable member propagation once Clang stops emitting PTR_AND_OBJ and uses
10921 // attach-style maps throughout.
10922 uint64_t ModifierBits =
10923 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10924 OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS |
10925 OpenMPOffloadMappingFlags::OMP_MAP_DELETE |
10926 OpenMPOffloadMappingFlags::OMP_MAP_CLOSE);
10927 if (PropagatePresentToPointee && Info->HasAttachPtr[I])
10928 ModifierBits |=
10929 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10930 OpenMPOffloadMappingFlags::OMP_MAP_PRESENT);
10931 Value *ImportedModifierBits =
10932 Builder.CreateAnd(LHS: MapType, RHS: Builder.getInt64(C: ModifierBits));
10933 Value *CurMapTypeWithModifiers = Builder.CreateOr(
10934 LHS: CurMapType, RHS: ImportedModifierBits, Name: "omp.maptype.with.modifiers");
10935
10936 // ATTACH entries must not receive map-type-modifying bits: ATTACH|ALWAYS is
10937 // reserved for the attach(always) map-type modifier, and other modifier
10938 // bits (DELETE, CLOSE) have no meaning for an ATTACH entry.
10939 Value *FinalMapType =
10940 (RawType & AttachBit) ? CurMapType : CurMapTypeWithModifiers;
10941
10942 Value *OffloadingArgs[] = {MapperHandle, CurBaseArg, CurBeginArg,
10943 CurSizeArg, FinalMapType, CurNameArg};
10944
10945 auto ChildMapperFn = CustomMapperCB(I);
10946 if (!ChildMapperFn)
10947 return ChildMapperFn.takeError();
10948 if (*ChildMapperFn) {
10949 // Call the corresponding mapper function.
10950 createRuntimeFunctionCall(Callee: *ChildMapperFn, Args: OffloadingArgs)
10951 ->setDoesNotThrow();
10952 } else {
10953 // Call the runtime API __tgt_push_mapper_component to fill up the runtime
10954 // data structure.
10955 createRuntimeFunctionCall(
10956 Callee: getOrCreateRuntimeFunction(M, FnID: OMPRTL___tgt_push_mapper_component),
10957 Args: OffloadingArgs);
10958 }
10959 }
10960
10961 // Update the pointer to point to the next element that needs to be mapped,
10962 // and check whether we have mapped all elements.
10963 Value *PtrNext = Builder.CreateConstGEP1_32(Ty: ElemTy, Ptr: PtrPHI, /*Idx0=*/1,
10964 Name: "omp.arraymap.next");
10965 PtrPHI->addIncoming(V: PtrNext, BB: LastBB);
10966 Value *IsDone = Builder.CreateICmpEQ(LHS: PtrNext, RHS: PtrEnd, Name: "omp.arraymap.isdone");
10967 BasicBlock *ExitBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp.arraymap.exit");
10968 Builder.CreateCondBr(Cond: IsDone, True: ExitBB, False: BodyBB);
10969
10970 emitBlock(BB: ExitBB, CurFn: MapperFn);
10971 // Emit array deletion if this is an array section and \p MapType indicates
10972 // that deletion is required.
10973 emitUDMapperArrayInitOrDel(MapperFn, MapperHandle, Base: BaseIn, Begin: BeginIn, Size,
10974 MapType, MapName, ElementSize, ExitBB: DoneBB,
10975 /*IsInit=*/false);
10976
10977 // Emit the function exit block.
10978 emitBlock(BB: DoneBB, CurFn: MapperFn, /*IsFinished=*/true);
10979
10980 Builder.CreateRetVoid();
10981 return MapperFn;
10982}
10983
10984Error OpenMPIRBuilder::emitOffloadingArrays(
10985 InsertPointTy AllocaIP, InsertPointTy CodeGenIP, MapInfosTy &CombinedInfo,
10986 TargetDataInfo &Info, CustomMapperCallbackTy CustomMapperCB,
10987 bool IsNonContiguous,
10988 function_ref<void(unsigned int, Value *)> DeviceAddrCB) {
10989
10990 // Reset the array information.
10991 Info.clearArrayInfo();
10992 Info.NumberOfPtrs = CombinedInfo.BasePointers.size();
10993
10994 if (Info.NumberOfPtrs == 0)
10995 return Error::success();
10996
10997 Builder.restoreIP(IP: AllocaIP);
10998 // Detect if we have any capture size requiring runtime evaluation of the
10999 // size so that a constant array could be eventually used.
11000 ArrayType *PointerArrayType =
11001 ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: Info.NumberOfPtrs);
11002
11003 Info.RTArgs.BasePointersArray = Builder.CreateAlloca(
11004 Ty: PointerArrayType, /* ArraySize = */ nullptr, Name: ".offload_baseptrs");
11005
11006 Info.RTArgs.PointersArray = Builder.CreateAlloca(
11007 Ty: PointerArrayType, /* ArraySize = */ nullptr, Name: ".offload_ptrs");
11008 AllocaInst *MappersArray = Builder.CreateAlloca(
11009 Ty: PointerArrayType, /* ArraySize = */ nullptr, Name: ".offload_mappers");
11010 Info.RTArgs.MappersArray = MappersArray;
11011
11012 // If we don't have any VLA types or other types that require runtime
11013 // evaluation, we can use a constant array for the map sizes, otherwise we
11014 // need to fill up the arrays as we do for the pointers.
11015 Type *Int64Ty = Builder.getInt64Ty();
11016 SmallVector<Constant *> ConstSizes(CombinedInfo.Sizes.size(),
11017 ConstantInt::get(Ty: Int64Ty, V: 0));
11018 SmallBitVector RuntimeSizes(CombinedInfo.Sizes.size());
11019 for (unsigned I = 0, E = CombinedInfo.Sizes.size(); I < E; ++I) {
11020 bool IsNonContigEntry =
11021 IsNonContiguous &&
11022 (static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
11023 CombinedInfo.Types[I] &
11024 OpenMPOffloadMappingFlags::OMP_MAP_NON_CONTIG) != 0);
11025 // For NON_CONTIG entries, ArgSizes stores the dimension count (number of
11026 // descriptor_dim records), not the byte size.
11027 if (IsNonContigEntry) {
11028 assert(I < CombinedInfo.NonContigInfo.Dims.size() &&
11029 "Index must be in-bounds for NON_CONTIG Dims array");
11030 const uint64_t DimCount = CombinedInfo.NonContigInfo.Dims[I];
11031 assert(DimCount > 0 && "NON_CONTIG DimCount must be > 0");
11032 ConstSizes[I] = ConstantInt::get(Ty: Int64Ty, V: DimCount);
11033 continue;
11034 }
11035 if (auto *CI = dyn_cast<Constant>(Val: CombinedInfo.Sizes[I])) {
11036 if (!isa<ConstantExpr>(Val: CI) && !isa<GlobalValue>(Val: CI)) {
11037 ConstSizes[I] = CI;
11038 continue;
11039 }
11040 }
11041 RuntimeSizes.set(I);
11042 }
11043
11044 if (RuntimeSizes.all()) {
11045 ArrayType *SizeArrayType = ArrayType::get(ElementType: Int64Ty, NumElements: Info.NumberOfPtrs);
11046 Info.RTArgs.SizesArray = Builder.CreateAlloca(
11047 Ty: SizeArrayType, /* ArraySize = */ nullptr, Name: ".offload_sizes");
11048 restoreIPandDebugLoc(Builder, IP: CodeGenIP);
11049 } else {
11050 auto *SizesArrayInit = ConstantArray::get(
11051 T: ArrayType::get(ElementType: Int64Ty, NumElements: ConstSizes.size()), V: ConstSizes);
11052 std::string Name = createPlatformSpecificName(Parts: {"offload_sizes"});
11053 auto *SizesArrayGbl =
11054 new GlobalVariable(M, SizesArrayInit->getType(), /*isConstant=*/true,
11055 GlobalValue::PrivateLinkage, SizesArrayInit, Name);
11056 SizesArrayGbl->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
11057
11058 if (!RuntimeSizes.any()) {
11059 Info.RTArgs.SizesArray = SizesArrayGbl;
11060 } else {
11061 unsigned IndexSize = M.getDataLayout().getIndexSizeInBits(AS: 0);
11062 Align OffloadSizeAlign = M.getDataLayout().getABIIntegerTypeAlignment(BitWidth: 64);
11063 ArrayType *SizeArrayType = ArrayType::get(ElementType: Int64Ty, NumElements: Info.NumberOfPtrs);
11064 AllocaInst *Buffer = Builder.CreateAlloca(
11065 Ty: SizeArrayType, /* ArraySize = */ nullptr, Name: ".offload_sizes");
11066 Buffer->setAlignment(OffloadSizeAlign);
11067 restoreIPandDebugLoc(Builder, IP: CodeGenIP);
11068 Builder.CreateMemCpy(
11069 Dst: Buffer, DstAlign: M.getDataLayout().getPrefTypeAlign(Ty: Buffer->getType()),
11070 Src: SizesArrayGbl, SrcAlign: OffloadSizeAlign,
11071 Size: Builder.getIntN(
11072 N: IndexSize,
11073 C: Buffer->getAllocationSize(DL: M.getDataLayout())->getFixedValue()));
11074
11075 Info.RTArgs.SizesArray = Buffer;
11076 }
11077 restoreIPandDebugLoc(Builder, IP: CodeGenIP);
11078 }
11079
11080 // The map types are always constant so we don't need to generate code to
11081 // fill arrays. Instead, we create an array constant.
11082 SmallVector<uint64_t, 4> Mapping;
11083 for (auto mapFlag : CombinedInfo.Types)
11084 Mapping.push_back(
11085 Elt: static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
11086 mapFlag));
11087 std::string MaptypesName = createPlatformSpecificName(Parts: {"offload_maptypes"});
11088 auto *MapTypesArrayGbl = createOffloadMaptypes(Mappings&: Mapping, VarName: MaptypesName);
11089 Info.RTArgs.MapTypesArray = MapTypesArrayGbl;
11090
11091 // The information types are only built if provided.
11092 if (!CombinedInfo.Names.empty()) {
11093 auto *MapNamesArrayGbl = createOffloadMapnames(
11094 Names&: CombinedInfo.Names, VarName: createPlatformSpecificName(Parts: {"offload_mapnames"}));
11095 Info.RTArgs.MapNamesArray = MapNamesArrayGbl;
11096 Info.EmitDebug = true;
11097 } else {
11098 Info.RTArgs.MapNamesArray =
11099 Constant::getNullValue(Ty: PointerType::getUnqual(C&: Builder.getContext()));
11100 Info.EmitDebug = false;
11101 }
11102
11103 // If there's a present map type modifier, it must not be applied to the end
11104 // of a region, so generate a separate map type array in that case.
11105 if (Info.separateBeginEndCalls()) {
11106 bool EndMapTypesDiffer = false;
11107 for (uint64_t &Type : Mapping) {
11108 if (Type & static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
11109 OpenMPOffloadMappingFlags::OMP_MAP_PRESENT)) {
11110 Type &= ~static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
11111 OpenMPOffloadMappingFlags::OMP_MAP_PRESENT);
11112 EndMapTypesDiffer = true;
11113 }
11114 }
11115 if (EndMapTypesDiffer) {
11116 MapTypesArrayGbl = createOffloadMaptypes(Mappings&: Mapping, VarName: MaptypesName);
11117 Info.RTArgs.MapTypesArrayEnd = MapTypesArrayGbl;
11118 }
11119 }
11120
11121 PointerType *PtrTy = Builder.getPtrTy();
11122 for (unsigned I = 0; I < Info.NumberOfPtrs; ++I) {
11123 Value *BPVal = CombinedInfo.BasePointers[I];
11124 Value *BP = Builder.CreateConstInBoundsGEP2_32(
11125 Ty: ArrayType::get(ElementType: PtrTy, NumElements: Info.NumberOfPtrs), Ptr: Info.RTArgs.BasePointersArray,
11126 Idx0: 0, Idx1: I);
11127 Builder.CreateAlignedStore(Val: BPVal, Ptr: BP,
11128 Align: M.getDataLayout().getPrefTypeAlign(Ty: PtrTy));
11129
11130 if (Info.requiresDevicePointerInfo()) {
11131 if (CombinedInfo.DevicePointers[I] == DeviceInfoTy::Pointer) {
11132 CodeGenIP = Builder.saveIP();
11133 Builder.restoreIP(IP: AllocaIP);
11134 Info.DevicePtrInfoMap[BPVal] = {BP, Builder.CreateAlloca(Ty: PtrTy)};
11135 restoreIPandDebugLoc(Builder, IP: CodeGenIP);
11136 if (DeviceAddrCB)
11137 DeviceAddrCB(I, Info.DevicePtrInfoMap[BPVal].second);
11138 } else if (CombinedInfo.DevicePointers[I] == DeviceInfoTy::Address) {
11139 Info.DevicePtrInfoMap[BPVal] = {BP, BP};
11140 if (DeviceAddrCB)
11141 DeviceAddrCB(I, BP);
11142 }
11143 }
11144
11145 Value *PVal = CombinedInfo.Pointers[I];
11146 Value *P = Builder.CreateConstInBoundsGEP2_32(
11147 Ty: ArrayType::get(ElementType: PtrTy, NumElements: Info.NumberOfPtrs), Ptr: Info.RTArgs.PointersArray, Idx0: 0,
11148 Idx1: I);
11149 // TODO: Check alignment correct.
11150 Builder.CreateAlignedStore(Val: PVal, Ptr: P,
11151 Align: M.getDataLayout().getPrefTypeAlign(Ty: PtrTy));
11152
11153 if (RuntimeSizes.test(Idx: I)) {
11154 Value *S = Builder.CreateConstInBoundsGEP2_32(
11155 Ty: ArrayType::get(ElementType: Int64Ty, NumElements: Info.NumberOfPtrs), Ptr: Info.RTArgs.SizesArray,
11156 /*Idx0=*/0,
11157 /*Idx1=*/I);
11158 Builder.CreateAlignedStore(Val: Builder.CreateIntCast(V: CombinedInfo.Sizes[I],
11159 DestTy: Int64Ty,
11160 /*isSigned=*/true),
11161 Ptr: S, Align: M.getDataLayout().getPrefTypeAlign(Ty: PtrTy));
11162 }
11163 // Fill up the mapper array.
11164 unsigned IndexSize = M.getDataLayout().getIndexSizeInBits(AS: 0);
11165 Value *MFunc = ConstantPointerNull::get(T: PtrTy);
11166
11167 auto CustomMFunc = CustomMapperCB(I);
11168 if (!CustomMFunc)
11169 return CustomMFunc.takeError();
11170 if (*CustomMFunc)
11171 MFunc = Builder.CreatePointerCast(V: *CustomMFunc, DestTy: PtrTy);
11172
11173 Value *MAddr = Builder.CreateInBoundsGEP(
11174 Ty: PointerArrayType, Ptr: MappersArray,
11175 IdxList: {Builder.getIntN(N: IndexSize, C: 0), Builder.getIntN(N: IndexSize, C: I)});
11176 Builder.CreateAlignedStore(
11177 Val: MFunc, Ptr: MAddr, Align: M.getDataLayout().getPrefTypeAlign(Ty: MAddr->getType()));
11178 }
11179
11180 if (!IsNonContiguous || CombinedInfo.NonContigInfo.Offsets.empty() ||
11181 Info.NumberOfPtrs == 0)
11182 return Error::success();
11183 emitNonContiguousDescriptor(AllocaIP, CodeGenIP, CombinedInfo, Info);
11184 return Error::success();
11185}
11186
11187void OpenMPIRBuilder::emitBranch(BasicBlock *Target) {
11188 BasicBlock *CurBB = Builder.GetInsertBlock();
11189
11190 if (!CurBB || CurBB->hasTerminator()) {
11191 // If there is no insert point or the previous block is already
11192 // terminated, don't touch it.
11193 } else {
11194 // Otherwise, create a fall-through branch.
11195 Builder.CreateBr(Dest: Target);
11196 }
11197
11198 Builder.ClearInsertionPoint();
11199}
11200
11201void OpenMPIRBuilder::emitBlock(BasicBlock *BB, Function *CurFn,
11202 bool IsFinished) {
11203 BasicBlock *CurBB = Builder.GetInsertBlock();
11204
11205 // Fall out of the current block (if necessary).
11206 emitBranch(Target: BB);
11207
11208 if (IsFinished && BB->use_empty()) {
11209 BB->eraseFromParent();
11210 return;
11211 }
11212
11213 // Place the block after the current block, if possible, or else at
11214 // the end of the function.
11215 if (CurBB && CurBB->getParent())
11216 CurFn->insert(Position: std::next(x: CurBB->getIterator()), BB);
11217 else
11218 CurFn->insert(Position: CurFn->end(), BB);
11219 Builder.SetInsertPoint(BB);
11220}
11221
11222Error OpenMPIRBuilder::emitIfClause(Value *Cond, BodyGenCallbackTy ThenGen,
11223 BodyGenCallbackTy ElseGen,
11224 InsertPointTy AllocaIP,
11225 ArrayRef<BasicBlock *> DeallocBlocks) {
11226 // If the condition constant folds and can be elided, try to avoid emitting
11227 // the condition and the dead arm of the if/else.
11228 if (auto *CI = dyn_cast<ConstantInt>(Val: Cond)) {
11229 auto CondConstant = CI->getSExtValue();
11230 if (CondConstant)
11231 return ThenGen(AllocaIP, Builder.saveIP(), DeallocBlocks);
11232
11233 return ElseGen(AllocaIP, Builder.saveIP(), DeallocBlocks);
11234 }
11235
11236 Function *CurFn = Builder.GetInsertBlock()->getParent();
11237
11238 // Otherwise, the condition did not fold, or we couldn't elide it. Just
11239 // emit the conditional branch.
11240 BasicBlock *ThenBlock = BasicBlock::Create(Context&: M.getContext(), Name: "omp_if.then");
11241 BasicBlock *ElseBlock = BasicBlock::Create(Context&: M.getContext(), Name: "omp_if.else");
11242 BasicBlock *ContBlock = BasicBlock::Create(Context&: M.getContext(), Name: "omp_if.end");
11243 Builder.CreateCondBr(Cond, True: ThenBlock, False: ElseBlock);
11244 // Emit the 'then' code.
11245 emitBlock(BB: ThenBlock, CurFn);
11246 if (Error Err = ThenGen(AllocaIP, Builder.saveIP(), DeallocBlocks))
11247 return Err;
11248 emitBranch(Target: ContBlock);
11249 // Emit the 'else' code if present.
11250 // There is no need to emit line number for unconditional branch.
11251 emitBlock(BB: ElseBlock, CurFn);
11252 if (Error Err = ElseGen(AllocaIP, Builder.saveIP(), DeallocBlocks))
11253 return Err;
11254 // There is no need to emit line number for unconditional branch.
11255 emitBranch(Target: ContBlock);
11256 // Emit the continuation block for code after the if.
11257 emitBlock(BB: ContBlock, CurFn, /*IsFinished=*/true);
11258 return Error::success();
11259}
11260
11261bool OpenMPIRBuilder::checkAndEmitFlushAfterAtomic(
11262 const LocationDescription &Loc, llvm::AtomicOrdering AO, AtomicKind AK) {
11263 assert(!(AO == AtomicOrdering::NotAtomic ||
11264 AO == llvm::AtomicOrdering::Unordered) &&
11265 "Unexpected Atomic Ordering.");
11266
11267 bool Flush = false;
11268 llvm::AtomicOrdering FlushAO = AtomicOrdering::Monotonic;
11269
11270 switch (AK) {
11271 case Read:
11272 if (AO == AtomicOrdering::Acquire || AO == AtomicOrdering::AcquireRelease ||
11273 AO == AtomicOrdering::SequentiallyConsistent) {
11274 FlushAO = AtomicOrdering::Acquire;
11275 Flush = true;
11276 }
11277 break;
11278 case Write:
11279 case Compare:
11280 case Update:
11281 if (AO == AtomicOrdering::Release || AO == AtomicOrdering::AcquireRelease ||
11282 AO == AtomicOrdering::SequentiallyConsistent) {
11283 FlushAO = AtomicOrdering::Release;
11284 Flush = true;
11285 }
11286 break;
11287 case Capture:
11288 switch (AO) {
11289 case AtomicOrdering::Acquire:
11290 FlushAO = AtomicOrdering::Acquire;
11291 Flush = true;
11292 break;
11293 case AtomicOrdering::Release:
11294 FlushAO = AtomicOrdering::Release;
11295 Flush = true;
11296 break;
11297 case AtomicOrdering::AcquireRelease:
11298 case AtomicOrdering::SequentiallyConsistent:
11299 FlushAO = AtomicOrdering::AcquireRelease;
11300 Flush = true;
11301 break;
11302 default:
11303 // do nothing - leave silently.
11304 break;
11305 }
11306 }
11307
11308 if (Flush) {
11309 // Currently Flush RT call still doesn't take memory_ordering, so for when
11310 // that happens, this tries to do the resolution of which atomic ordering
11311 // to use with but issue the flush call
11312 // TODO: pass `FlushAO` after memory ordering support is added
11313 (void)FlushAO;
11314 emitFlush(Loc);
11315 }
11316
11317 // for AO == AtomicOrdering::Monotonic and all other case combinations
11318 // do nothing
11319 return Flush;
11320}
11321
11322OpenMPIRBuilder::InsertPointTy
11323OpenMPIRBuilder::createAtomicRead(const LocationDescription &Loc,
11324 AtomicOpValue &X, AtomicOpValue &V,
11325 AtomicOrdering AO, InsertPointTy AllocaIP) {
11326 if (!updateToLocation(Loc))
11327 return Loc.IP;
11328
11329 assert(X.Var->getType()->isPointerTy() &&
11330 "OMP Atomic expects a pointer to target memory");
11331 Type *XElemTy = X.ElemTy;
11332 assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||
11333 XElemTy->isPointerTy() || XElemTy->isStructTy()) &&
11334 "OMP atomic read expected a scalar type");
11335
11336 Value *XRead = nullptr;
11337
11338 if (XElemTy->isIntegerTy()) {
11339 LoadInst *XLD =
11340 Builder.CreateLoad(Ty: XElemTy, Ptr: X.Var, isVolatile: X.IsVolatile, Name: "omp.atomic.read");
11341 XLD->setAtomic(Ordering: AO);
11342 XRead = cast<Value>(Val: XLD);
11343 } else if (XElemTy->isStructTy()) {
11344 // FIXME: Add checks to ensure __atomic_load is emitted iff the
11345 // target does not support `atomicrmw` of the size of the struct
11346 LoadInst *OldVal = Builder.CreateLoad(Ty: XElemTy, Ptr: X.Var, Name: "omp.atomic.read");
11347 OldVal->setAtomic(Ordering: AO);
11348 const DataLayout &DL = OldVal->getModule()->getDataLayout();
11349 unsigned LoadSize = DL.getTypeStoreSize(Ty: XElemTy);
11350 OpenMPIRBuilder::AtomicInfo atomicInfo(
11351 &Builder, XElemTy, LoadSize * 8, LoadSize * 8, OldVal->getAlign(),
11352 OldVal->getAlign(), true /* UseLibcall */, AllocaIP, X.Var);
11353 auto AtomicLoadRes = atomicInfo.EmitAtomicLoadLibcall(AO);
11354 XRead = AtomicLoadRes.first;
11355 OldVal->eraseFromParent();
11356 } else {
11357 // We need to perform atomic op as integer
11358 IntegerType *IntCastTy =
11359 IntegerType::get(C&: M.getContext(), NumBits: XElemTy->getScalarSizeInBits());
11360 LoadInst *XLoad =
11361 Builder.CreateLoad(Ty: IntCastTy, Ptr: X.Var, isVolatile: X.IsVolatile, Name: "omp.atomic.load");
11362 XLoad->setAtomic(Ordering: AO);
11363 if (XElemTy->isFloatingPointTy()) {
11364 XRead = Builder.CreateBitCast(V: XLoad, DestTy: XElemTy, Name: "atomic.flt.cast");
11365 } else {
11366 XRead = Builder.CreateIntToPtr(V: XLoad, DestTy: XElemTy, Name: "atomic.ptr.cast");
11367 }
11368 }
11369 checkAndEmitFlushAfterAtomic(Loc, AO, AK: AtomicKind::Read);
11370 Builder.CreateStore(Val: XRead, Ptr: V.Var, isVolatile: V.IsVolatile);
11371 return Builder.saveIP();
11372}
11373
11374OpenMPIRBuilder::InsertPointTy
11375OpenMPIRBuilder::createAtomicWrite(const LocationDescription &Loc,
11376 AtomicOpValue &X, Value *Expr,
11377 AtomicOrdering AO, InsertPointTy AllocaIP) {
11378 if (!updateToLocation(Loc))
11379 return Loc.IP;
11380
11381 assert(X.Var->getType()->isPointerTy() &&
11382 "OMP Atomic expects a pointer to target memory");
11383 Type *XElemTy = X.ElemTy;
11384 assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||
11385 XElemTy->isPointerTy() || XElemTy->isStructTy()) &&
11386 "OMP atomic write expected a scalar type");
11387
11388 if (XElemTy->isIntegerTy()) {
11389 StoreInst *XSt = Builder.CreateStore(Val: Expr, Ptr: X.Var, isVolatile: X.IsVolatile);
11390 XSt->setAtomic(Ordering: AO);
11391 } else if (XElemTy->isStructTy()) {
11392 LoadInst *OldVal = Builder.CreateLoad(Ty: XElemTy, Ptr: X.Var, Name: "omp.atomic.read");
11393 const DataLayout &DL = OldVal->getModule()->getDataLayout();
11394 unsigned LoadSize = DL.getTypeStoreSize(Ty: XElemTy);
11395 OpenMPIRBuilder::AtomicInfo atomicInfo(
11396 &Builder, XElemTy, LoadSize * 8, LoadSize * 8, OldVal->getAlign(),
11397 OldVal->getAlign(), true /* UseLibcall */, AllocaIP, X.Var);
11398 atomicInfo.EmitAtomicStoreLibcall(AO, Source: Expr);
11399 OldVal->eraseFromParent();
11400 } else {
11401 // We need to bitcast and perform atomic op as integers
11402 IntegerType *IntCastTy =
11403 IntegerType::get(C&: M.getContext(), NumBits: XElemTy->getScalarSizeInBits());
11404 Value *ExprCast =
11405 Builder.CreateBitCast(V: Expr, DestTy: IntCastTy, Name: "atomic.src.int.cast");
11406 StoreInst *XSt = Builder.CreateStore(Val: ExprCast, Ptr: X.Var, isVolatile: X.IsVolatile);
11407 XSt->setAtomic(Ordering: AO);
11408 }
11409
11410 checkAndEmitFlushAfterAtomic(Loc, AO, AK: AtomicKind::Write);
11411 return Builder.saveIP();
11412}
11413
11414OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createAtomicUpdate(
11415 const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X,
11416 Value *Expr, AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp,
11417 AtomicUpdateCallbackTy &UpdateOp, bool IsXBinopExpr,
11418 bool IsIgnoreDenormalMode, bool IsFineGrainedMemory, bool IsRemoteMemory) {
11419 assert(!isConflictIP(Loc.IP, AllocaIP) && "IPs must not be ambiguous");
11420 if (!updateToLocation(Loc))
11421 return Loc.IP;
11422
11423 LLVM_DEBUG({
11424 Type *XTy = X.Var->getType();
11425 assert(XTy->isPointerTy() &&
11426 "OMP Atomic expects a pointer to target memory");
11427 Type *XElemTy = X.ElemTy;
11428 assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||
11429 XElemTy->isPointerTy() || XElemTy->isStructTy()) &&
11430 "OMP atomic update expected a scalar or struct type");
11431 assert((RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) &&
11432 (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) &&
11433 "OpenMP atomic does not support LT or GT operations");
11434 });
11435
11436 Expected<std::pair<Value *, Value *>> AtomicResult = emitAtomicUpdate(
11437 AllocaIP, X: X.Var, XElemTy: X.ElemTy, Expr, AO, RMWOp, UpdateOp, VolatileX: X.IsVolatile,
11438 IsXBinopExpr, IsIgnoreDenormalMode, IsFineGrainedMemory, IsRemoteMemory);
11439 if (!AtomicResult)
11440 return AtomicResult.takeError();
11441 checkAndEmitFlushAfterAtomic(Loc, AO, AK: AtomicKind::Update);
11442 return Builder.saveIP();
11443}
11444
11445// FIXME: Duplicating AtomicExpand
11446Value *OpenMPIRBuilder::emitRMWOpAsInstruction(Value *Src1, Value *Src2,
11447 AtomicRMWInst::BinOp RMWOp) {
11448 switch (RMWOp) {
11449 case AtomicRMWInst::Add:
11450 return Builder.CreateAdd(LHS: Src1, RHS: Src2);
11451 case AtomicRMWInst::Sub:
11452 return Builder.CreateSub(LHS: Src1, RHS: Src2);
11453 case AtomicRMWInst::And:
11454 return Builder.CreateAnd(LHS: Src1, RHS: Src2);
11455 case AtomicRMWInst::Nand:
11456 return Builder.CreateNeg(V: Builder.CreateAnd(LHS: Src1, RHS: Src2));
11457 case AtomicRMWInst::Or:
11458 return Builder.CreateOr(LHS: Src1, RHS: Src2);
11459 case AtomicRMWInst::Xor:
11460 return Builder.CreateXor(LHS: Src1, RHS: Src2);
11461 case AtomicRMWInst::Xchg:
11462 case AtomicRMWInst::FAdd:
11463 case AtomicRMWInst::FSub:
11464 case AtomicRMWInst::BAD_BINOP:
11465 case AtomicRMWInst::Max:
11466 case AtomicRMWInst::Min:
11467 case AtomicRMWInst::UMax:
11468 case AtomicRMWInst::UMin:
11469 case AtomicRMWInst::FMax:
11470 case AtomicRMWInst::FMin:
11471 case AtomicRMWInst::FMaximum:
11472 case AtomicRMWInst::FMinimum:
11473 case AtomicRMWInst::FMaximumNum:
11474 case AtomicRMWInst::FMinimumNum:
11475 case AtomicRMWInst::UIncWrap:
11476 case AtomicRMWInst::UDecWrap:
11477 case AtomicRMWInst::USubCond:
11478 case AtomicRMWInst::USubSat:
11479 llvm_unreachable("Unsupported atomic update operation");
11480 }
11481 llvm_unreachable("Unsupported atomic update operation");
11482}
11483
11484static AtomicOrdering TransformReleaseAcquireRelease(AtomicOrdering AO) {
11485 // Loads cannot use Release or AcquireRelease ordering. This load is
11486 // just the initial value for the cmpxchg loop; the cmpxchg itself
11487 // retains the original ordering.
11488 AtomicOrdering LoadAO = AO;
11489
11490 if (AO == AtomicOrdering::Release) {
11491 LoadAO = AtomicOrdering::Monotonic;
11492 } else if (AO == AtomicOrdering::AcquireRelease) {
11493 LoadAO = AtomicOrdering::Acquire;
11494 }
11495
11496 return LoadAO;
11497}
11498
11499Expected<std::pair<Value *, Value *>> OpenMPIRBuilder::emitAtomicUpdate(
11500 InsertPointTy AllocaIP, Value *X, Type *XElemTy, Value *Expr,
11501 AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp,
11502 AtomicUpdateCallbackTy &UpdateOp, bool VolatileX, bool IsXBinopExpr,
11503 bool IsIgnoreDenormalMode, bool IsFineGrainedMemory, bool IsRemoteMemory) {
11504 // TODO: handle the case where XElemTy is not byte-sized or not a power of 2.
11505 bool emitRMWOp = false;
11506 switch (RMWOp) {
11507 case AtomicRMWInst::Add:
11508 case AtomicRMWInst::And:
11509 case AtomicRMWInst::Nand:
11510 case AtomicRMWInst::Or:
11511 case AtomicRMWInst::Xor:
11512 case AtomicRMWInst::Xchg:
11513 emitRMWOp = XElemTy;
11514 break;
11515 case AtomicRMWInst::Sub:
11516 emitRMWOp = (IsXBinopExpr && XElemTy);
11517 break;
11518 default:
11519 emitRMWOp = false;
11520 }
11521 emitRMWOp &= XElemTy->isIntegerTy();
11522
11523 std::pair<Value *, Value *> Res;
11524 if (emitRMWOp) {
11525 AtomicRMWInst *RMWInst =
11526 Builder.CreateAtomicRMW(Op: RMWOp, Ptr: X, Val: Expr, Align: llvm::MaybeAlign(), Ordering: AO);
11527 if (IsIgnoreDenormalMode)
11528 RMWInst->setMetadata(KindID: llvm::LLVMContext::MD_atomic_ignore_denormal_mode,
11529 Node: llvm::MDNode::get(Context&: Builder.getContext(), MDs: {}));
11530 if (T.isAMDGPU()) {
11531 if (!IsFineGrainedMemory)
11532 RMWInst->setMetadata(Kind: "amdgpu.no.fine.grained.memory",
11533 Node: llvm::MDNode::get(Context&: Builder.getContext(), MDs: {}));
11534 if (!IsRemoteMemory)
11535 RMWInst->setMetadata(Kind: "amdgpu.no.remote.memory",
11536 Node: llvm::MDNode::get(Context&: Builder.getContext(), MDs: {}));
11537 }
11538 Res.first = RMWInst;
11539 // not needed except in case of postfix captures. Generate anyway for
11540 // consistency with the else part. Will be removed with any DCE pass.
11541 // AtomicRMWInst::Xchg does not have a coressponding instruction.
11542 if (RMWOp == AtomicRMWInst::Xchg)
11543 Res.second = Res.first;
11544 else
11545 Res.second = emitRMWOpAsInstruction(Src1: Res.first, Src2: Expr, RMWOp);
11546 } else if (XElemTy->isStructTy()) {
11547 LoadInst *OldVal =
11548 Builder.CreateLoad(Ty: XElemTy, Ptr: X, Name: X->getName() + ".atomic.load");
11549 AtomicOrdering LoadAO = TransformReleaseAcquireRelease(AO);
11550 OldVal->setAtomic(Ordering: LoadAO);
11551 const DataLayout &LoadDL = OldVal->getModule()->getDataLayout();
11552 unsigned LoadSize = LoadDL.getTypeStoreSize(Ty: XElemTy);
11553
11554 OpenMPIRBuilder::AtomicInfo atomicInfo(
11555 &Builder, XElemTy, LoadSize * 8, LoadSize * 8, OldVal->getAlign(),
11556 OldVal->getAlign(), true /* UseLibcall */, AllocaIP, X);
11557 auto AtomicLoadRes = atomicInfo.EmitAtomicLoadLibcall(AO);
11558 BasicBlock *CurBB = Builder.GetInsertBlock();
11559 Instruction *CurBBTI = CurBB->getTerminatorOrNull();
11560 CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable();
11561 BasicBlock *ExitBB =
11562 CurBB->splitBasicBlock(I: CurBBTI, BBName: X->getName() + ".atomic.exit");
11563 BasicBlock *ContBB = CurBB->splitBasicBlock(I: CurBB->getTerminator(),
11564 BBName: X->getName() + ".atomic.cont");
11565 ContBB->getTerminator()->eraseFromParent();
11566 Builder.restoreIP(IP: AllocaIP);
11567 AllocaInst *NewAtomicAddr = Builder.CreateAlloca(Ty: XElemTy);
11568 NewAtomicAddr->setName(X->getName() + "x.new.val");
11569 Builder.SetInsertPoint(ContBB);
11570 llvm::PHINode *PHI = Builder.CreatePHI(Ty: OldVal->getType(), NumReservedValues: 2);
11571 PHI->addIncoming(V: AtomicLoadRes.first, BB: CurBB);
11572 Value *OldExprVal = PHI;
11573 Expected<Value *> CBResult = UpdateOp(OldExprVal, Builder);
11574 if (!CBResult)
11575 return CBResult.takeError();
11576 Value *Upd = *CBResult;
11577 Builder.CreateStore(Val: Upd, Ptr: NewAtomicAddr);
11578 AtomicOrdering Failure =
11579 llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: AO);
11580 auto Result = atomicInfo.EmitAtomicCompareExchangeLibcall(
11581 ExpectedVal: AtomicLoadRes.second, DesiredVal: NewAtomicAddr, Success: AO, Failure);
11582 LoadInst *PHILoad = Builder.CreateLoad(Ty: XElemTy, Ptr: Result.first);
11583 PHI->addIncoming(V: PHILoad, BB: Builder.GetInsertBlock());
11584 Builder.CreateCondBr(Cond: Result.second, True: ExitBB, False: ContBB);
11585 OldVal->eraseFromParent();
11586 Res.first = OldExprVal;
11587 Res.second = Upd;
11588
11589 if (UnreachableInst *ExitTI =
11590 dyn_cast<UnreachableInst>(Val: ExitBB->getTerminator())) {
11591 CurBBTI->eraseFromParent();
11592 Builder.SetInsertPoint(ExitBB);
11593 } else {
11594 Builder.SetInsertPoint(ExitTI);
11595 }
11596 } else {
11597 IntegerType *IntCastTy =
11598 IntegerType::get(C&: M.getContext(), NumBits: XElemTy->getScalarSizeInBits());
11599 LoadInst *OldVal =
11600 Builder.CreateLoad(Ty: IntCastTy, Ptr: X, Name: X->getName() + ".atomic.load");
11601 AtomicOrdering LoadAO = TransformReleaseAcquireRelease(AO);
11602 OldVal->setAtomic(Ordering: LoadAO);
11603 // CurBB
11604 // | /---\
11605 // ContBB |
11606 // | \---/
11607 // ExitBB
11608 BasicBlock *CurBB = Builder.GetInsertBlock();
11609 Instruction *CurBBTI = CurBB->getTerminatorOrNull();
11610 CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable();
11611 BasicBlock *ExitBB =
11612 CurBB->splitBasicBlock(I: CurBBTI, BBName: X->getName() + ".atomic.exit");
11613 BasicBlock *ContBB = CurBB->splitBasicBlock(I: CurBB->getTerminator(),
11614 BBName: X->getName() + ".atomic.cont");
11615 ContBB->getTerminator()->eraseFromParent();
11616 Builder.restoreIP(IP: AllocaIP);
11617 AllocaInst *NewAtomicAddr = Builder.CreateAlloca(Ty: XElemTy);
11618 NewAtomicAddr->setName(X->getName() + "x.new.val");
11619 Builder.SetInsertPoint(ContBB);
11620 llvm::PHINode *PHI = Builder.CreatePHI(Ty: OldVal->getType(), NumReservedValues: 2);
11621 PHI->addIncoming(V: OldVal, BB: CurBB);
11622 bool IsIntTy = XElemTy->isIntegerTy();
11623 Value *OldExprVal = PHI;
11624 if (!IsIntTy) {
11625 if (XElemTy->isFloatingPointTy()) {
11626 OldExprVal = Builder.CreateBitCast(V: PHI, DestTy: XElemTy,
11627 Name: X->getName() + ".atomic.fltCast");
11628 } else {
11629 OldExprVal = Builder.CreateIntToPtr(V: PHI, DestTy: XElemTy,
11630 Name: X->getName() + ".atomic.ptrCast");
11631 }
11632 }
11633
11634 Expected<Value *> CBResult = UpdateOp(OldExprVal, Builder);
11635 if (!CBResult)
11636 return CBResult.takeError();
11637 Value *Upd = *CBResult;
11638 Builder.CreateStore(Val: Upd, Ptr: NewAtomicAddr);
11639 LoadInst *DesiredVal = Builder.CreateLoad(Ty: IntCastTy, Ptr: NewAtomicAddr);
11640 AtomicOrdering Failure =
11641 llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: AO);
11642 AtomicCmpXchgInst *Result = Builder.CreateAtomicCmpXchg(
11643 Ptr: X, Cmp: PHI, New: DesiredVal, Align: llvm::MaybeAlign(), SuccessOrdering: AO, FailureOrdering: Failure);
11644 Result->setVolatile(VolatileX);
11645 Value *PreviousVal = Builder.CreateExtractValue(Agg: Result, /*Idxs=*/0);
11646 Value *SuccessFailureVal = Builder.CreateExtractValue(Agg: Result, /*Idxs=*/1);
11647 PHI->addIncoming(V: PreviousVal, BB: Builder.GetInsertBlock());
11648 Builder.CreateCondBr(Cond: SuccessFailureVal, True: ExitBB, False: ContBB);
11649
11650 Res.first = OldExprVal;
11651 Res.second = Upd;
11652
11653 // set Insertion point in exit block
11654 if (UnreachableInst *ExitTI =
11655 dyn_cast<UnreachableInst>(Val: ExitBB->getTerminator())) {
11656 CurBBTI->eraseFromParent();
11657 Builder.SetInsertPoint(ExitBB);
11658 } else {
11659 Builder.SetInsertPoint(ExitTI);
11660 }
11661 }
11662
11663 return Res;
11664}
11665
11666OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createAtomicCapture(
11667 const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X,
11668 AtomicOpValue &V, Value *Expr, AtomicOrdering AO,
11669 AtomicRMWInst::BinOp RMWOp, AtomicUpdateCallbackTy &UpdateOp,
11670 bool UpdateExpr, bool IsPostfixUpdate, bool IsXBinopExpr,
11671 bool IsIgnoreDenormalMode, bool IsFineGrainedMemory, bool IsRemoteMemory) {
11672 if (!updateToLocation(Loc))
11673 return Loc.IP;
11674
11675 LLVM_DEBUG({
11676 Type *XTy = X.Var->getType();
11677 assert(XTy->isPointerTy() &&
11678 "OMP Atomic expects a pointer to target memory");
11679 Type *XElemTy = X.ElemTy;
11680 assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||
11681 XElemTy->isPointerTy() || XElemTy->isStructTy()) &&
11682 "OMP atomic capture expected a scalar or struct type");
11683 assert((RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) &&
11684 "OpenMP atomic does not support LT or GT operations");
11685 });
11686
11687 // If UpdateExpr is 'x' updated with some `expr` not based on 'x',
11688 // 'x' is simply atomically rewritten with 'expr'.
11689 AtomicRMWInst::BinOp AtomicOp = (UpdateExpr ? RMWOp : AtomicRMWInst::Xchg);
11690 Expected<std::pair<Value *, Value *>> AtomicResult = emitAtomicUpdate(
11691 AllocaIP, X: X.Var, XElemTy: X.ElemTy, Expr, AO, RMWOp: AtomicOp, UpdateOp, VolatileX: X.IsVolatile,
11692 IsXBinopExpr, IsIgnoreDenormalMode, IsFineGrainedMemory, IsRemoteMemory);
11693 if (!AtomicResult)
11694 return AtomicResult.takeError();
11695 Value *CapturedVal =
11696 (IsPostfixUpdate ? AtomicResult->first : AtomicResult->second);
11697 Builder.CreateStore(Val: CapturedVal, Ptr: V.Var, isVolatile: V.IsVolatile);
11698
11699 checkAndEmitFlushAfterAtomic(Loc, AO, AK: AtomicKind::Capture);
11700 return Builder.saveIP();
11701}
11702
11703OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicCompare(
11704 const LocationDescription &Loc, AtomicOpValue &X, AtomicOpValue &V,
11705 AtomicOpValue &R, Value *E, Value *D, AtomicOrdering AO,
11706 omp::OMPAtomicCompareOp Op, bool IsXBinopExpr, bool IsPostfixUpdate,
11707 bool IsFailOnly, bool IsWeak) {
11708
11709 AtomicOrdering Failure = AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: AO);
11710 return createAtomicCompare(Loc, X, V, R, E, D, AO, Op, IsXBinopExpr,
11711 IsPostfixUpdate, IsFailOnly, Failure, IsWeak);
11712}
11713
11714OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicCompare(
11715 const LocationDescription &Loc, AtomicOpValue &X, AtomicOpValue &V,
11716 AtomicOpValue &R, Value *E, Value *D, AtomicOrdering AO,
11717 omp::OMPAtomicCompareOp Op, bool IsXBinopExpr, bool IsPostfixUpdate,
11718 bool IsFailOnly, AtomicOrdering Failure, bool IsWeak) {
11719
11720 if (!updateToLocation(Loc))
11721 return Loc.IP;
11722
11723 assert(X.Var->getType()->isPointerTy() &&
11724 "OMP atomic expects a pointer to target memory");
11725 // compare capture
11726 if (V.Var) {
11727 assert(V.Var->getType()->isPointerTy() && "v.var must be of pointer type");
11728 assert(V.ElemTy == X.ElemTy && "x and v must be of same type");
11729 }
11730
11731 bool IsInteger = E->getType()->isIntegerTy();
11732
11733 if (Op == OMPAtomicCompareOp::EQ) {
11734 // OldValue and SuccessOrFail are set below and used in the shared V.Var /
11735 // R.Var handling.
11736 Value *OldValue = nullptr;
11737 Value *SuccessOrFail = nullptr;
11738
11739 if (!IsInteger && HandleFPNegZero) {
11740 // IEEE 754 special cases for cmpxchg (which is bitwise):
11741 // 1. -0.0 == +0.0 but they have different bit patterns.
11742 // 2. NaN != NaN but identical NaN bit patterns would match.
11743 //
11744 // CurBB:
11745 // %e_int = bitcast E to intN
11746 // %d_int = bitcast D to intN
11747 // %x_curr = load atomic intN, X
11748 // %x_fp = bitcast %x_curr to FP
11749 // %e_is_nan = fcmp uno E, E
11750 // %x_is_nan = fcmp uno %x_fp, %x_fp
11751 // %either_nan = or %e_is_nan, %x_is_nan
11752 // br %either_nan, NaNBB, NotNaNBB
11753 // NaNBB: ; NaN == anything is always false
11754 // br ExitBB
11755 // NotNaNBB:
11756 // %x_is_zero = fcmp oeq %x_fp, 0.0
11757 // %e_is_zero = fcmp oeq E, 0.0
11758 // %both_zero = and %x_is_zero, %e_is_zero
11759 // br %both_zero, ZeroBB, NormalBB
11760 // ZeroBB: ; both ±0.0 → x = d
11761 // cmpxchg X, %x_curr, %d_int
11762 // br ExitBB
11763 // NormalBB: ; original path
11764 // cmpxchg X, %e_int, %d_int
11765 // br ExitBB
11766 // ExitBB:
11767 // phi merge
11768 IntegerType *IntCastTy =
11769 IntegerType::get(C&: M.getContext(), NumBits: X.ElemTy->getScalarSizeInBits());
11770 Value *EBCast = Builder.CreateBitCast(V: E, DestTy: IntCastTy);
11771 Value *DBCast = Builder.CreateBitCast(V: D, DestTy: IntCastTy);
11772
11773 // Load X atomically.
11774 LoadInst *XCurr = Builder.CreateLoad(Ty: IntCastTy, Ptr: X.Var,
11775 Name: X.Var->getName() + ".atomic.load");
11776 XCurr->setAtomic(Ordering: AtomicOrdering::Monotonic);
11777 Value *XFP = Builder.CreateBitCast(V: XCurr, DestTy: X.ElemTy);
11778
11779 // IEEE 754: NaN != NaN, but cmpxchg would succeed if E and X have
11780 // the same NaN bit pattern. Skip cmpxchg when either is NaN.
11781 Value *EIsNaN = Builder.CreateFCmpUNO(LHS: E, RHS: E, Name: "atomic.e.isnan");
11782 Value *XIsNaN = Builder.CreateFCmpUNO(LHS: XFP, RHS: XFP, Name: "atomic.x.isnan");
11783 Value *EitherNaN = Builder.CreateOr(LHS: EIsNaN, RHS: XIsNaN, Name: "atomic.either.nan");
11784
11785 BasicBlock *CurBB = Builder.GetInsertBlock();
11786 Function *F = CurBB->getParent();
11787 Instruction *CurBBTI = CurBB->getTerminatorOrNull();
11788 CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable();
11789 BasicBlock *ExitBB =
11790 CurBB->splitBasicBlock(I: CurBBTI, BBName: X.Var->getName() + ".atomic.exit");
11791 BasicBlock *NaNBB = BasicBlock::Create(
11792 Context&: M.getContext(), Name: X.Var->getName() + ".atomic.nan", Parent: F, InsertBefore: ExitBB);
11793 BasicBlock *NotNaNBB = BasicBlock::Create(
11794 Context&: M.getContext(), Name: X.Var->getName() + ".atomic.notnan", Parent: F, InsertBefore: ExitBB);
11795 BasicBlock *ZeroBB = BasicBlock::Create(
11796 Context&: M.getContext(), Name: X.Var->getName() + ".atomic.zero", Parent: F, InsertBefore: ExitBB);
11797 BasicBlock *NormalBB = BasicBlock::Create(
11798 Context&: M.getContext(), Name: X.Var->getName() + ".atomic.normal", Parent: F, InsertBefore: ExitBB);
11799
11800 // If either E or X is NaN → NaNBB (always fails), else check for ±0.0.
11801 CurBB->getTerminator()->eraseFromParent();
11802 Builder.SetInsertPoint(CurBB);
11803 Builder.CreateCondBr(Cond: EitherNaN, True: NaNBB, False: NotNaNBB);
11804
11805 // NaNBB: NaN == anything is always false; skip cmpxchg.
11806 Builder.SetInsertPoint(NaNBB);
11807 Builder.CreateBr(Dest: ExitBB);
11808
11809 // NotNaNBB: check both X and E for ±0.0.
11810 Builder.SetInsertPoint(NotNaNBB);
11811 Value *XIsZero =
11812 Builder.CreateFCmpOEQ(LHS: XFP, RHS: ConstantFP::getZero(Ty: X.ElemTy),
11813 Name: X.Var->getName() + ".atomic.xiszero");
11814 Value *EIsZero = Builder.CreateFCmpOEQ(LHS: E, RHS: ConstantFP::getZero(Ty: X.ElemTy),
11815 Name: "atomic.e.iszero");
11816 Value *BothZero = Builder.CreateAnd(LHS: XIsZero, RHS: EIsZero, Name: "atomic.both.zero");
11817 Builder.CreateCondBr(Cond: BothZero, True: ZeroBB, False: NormalBB);
11818
11819 // ZeroBB: cmpxchg with X's loaded bit-pattern.
11820 Builder.SetInsertPoint(ZeroBB);
11821 AtomicCmpXchgInst *ResZero = Builder.CreateAtomicCmpXchg(
11822 Ptr: X.Var, Cmp: XCurr, New: DBCast, Align: MaybeAlign(), SuccessOrdering: AO, FailureOrdering: Failure);
11823 ResZero->setWeak(IsWeak);
11824 Value *OldZero = Builder.CreateExtractValue(Agg: ResZero, /*Idxs=*/0);
11825 Value *OkZero = Builder.CreateExtractValue(Agg: ResZero, /*Idxs=*/1);
11826 Builder.CreateBr(Dest: ExitBB);
11827
11828 // NormalBB: original bitwise cmpxchg.
11829 Builder.SetInsertPoint(NormalBB);
11830 AtomicCmpXchgInst *ResNormal = Builder.CreateAtomicCmpXchg(
11831 Ptr: X.Var, Cmp: EBCast, New: DBCast, Align: MaybeAlign(), SuccessOrdering: AO, FailureOrdering: Failure);
11832 ResNormal->setWeak(IsWeak);
11833 Value *OldNormal = Builder.CreateExtractValue(Agg: ResNormal, /*Idxs=*/0);
11834 Value *OkNormal = Builder.CreateExtractValue(Agg: ResNormal, /*Idxs=*/1);
11835 Builder.CreateBr(Dest: ExitBB);
11836
11837 // ExitBB: merge results from NaN, Zero, and Normal paths.
11838 Builder.SetInsertPoint(TheBB: ExitBB, IP: ExitBB->begin());
11839 PHINode *OldIntPHI =
11840 Builder.CreatePHI(Ty: IntCastTy, NumReservedValues: 3, Name: X.Var->getName() + ".atomic.old");
11841 OldIntPHI->addIncoming(V: XCurr, BB: NaNBB);
11842 OldIntPHI->addIncoming(V: OldZero, BB: ZeroBB);
11843 OldIntPHI->addIncoming(V: OldNormal, BB: NormalBB);
11844 PHINode *SuccessPHI = Builder.CreatePHI(Ty: Builder.getInt1Ty(), NumReservedValues: 3,
11845 Name: X.Var->getName() + ".atomic.ok");
11846 SuccessPHI->addIncoming(V: Builder.getFalse(), BB: NaNBB);
11847 SuccessPHI->addIncoming(V: OkZero, BB: ZeroBB);
11848 SuccessPHI->addIncoming(V: OkNormal, BB: NormalBB);
11849
11850 if (isa<UnreachableInst>(Val: ExitBB->getTerminator())) {
11851 CurBBTI->eraseFromParent();
11852 Builder.SetInsertPoint(ExitBB);
11853 } else {
11854 Builder.SetInsertPoint(&*ExitBB->getFirstNonPHIIt());
11855 }
11856
11857 OldValue = Builder.CreateBitCast(V: OldIntPHI, DestTy: X.ElemTy,
11858 Name: X.Var->getName() + ".atomic.old.fp");
11859 SuccessOrFail = SuccessPHI;
11860 } else {
11861 AtomicCmpXchgInst *Result = nullptr;
11862 if (!IsInteger) {
11863 IntegerType *IntCastTy =
11864 IntegerType::get(C&: M.getContext(), NumBits: X.ElemTy->getScalarSizeInBits());
11865 Value *EBCast = Builder.CreateBitCast(V: E, DestTy: IntCastTy);
11866 Value *DBCast = Builder.CreateBitCast(V: D, DestTy: IntCastTy);
11867 Result = Builder.CreateAtomicCmpXchg(Ptr: X.Var, Cmp: EBCast, New: DBCast,
11868 Align: MaybeAlign(), SuccessOrdering: AO, FailureOrdering: Failure);
11869 } else {
11870 Result =
11871 Builder.CreateAtomicCmpXchg(Ptr: X.Var, Cmp: E, New: D, Align: MaybeAlign(), SuccessOrdering: AO, FailureOrdering: Failure);
11872 }
11873 Result->setWeak(IsWeak);
11874
11875 if (V.Var) {
11876 OldValue = Builder.CreateExtractValue(Agg: Result, /*Idxs=*/0);
11877 if (!IsInteger)
11878 OldValue = Builder.CreateBitCast(V: OldValue, DestTy: X.ElemTy);
11879 assert(OldValue->getType() == V.ElemTy &&
11880 "OldValue and V must be of same type");
11881 if (IsPostfixUpdate) {
11882 Builder.CreateStore(Val: OldValue, Ptr: V.Var, isVolatile: V.IsVolatile);
11883 } else {
11884 SuccessOrFail = Builder.CreateExtractValue(Agg: Result, /*Idxs=*/1);
11885 if (IsFailOnly) {
11886 BasicBlock *CurBB = Builder.GetInsertBlock();
11887 Instruction *CurBBTI = CurBB->getTerminatorOrNull();
11888 CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable();
11889 BasicBlock *ExitBB = CurBB->splitBasicBlock(
11890 I: CurBBTI, BBName: X.Var->getName() + ".atomic.exit");
11891 BasicBlock *ContBB = CurBB->splitBasicBlock(
11892 I: CurBB->getTerminator(), BBName: X.Var->getName() + ".atomic.cont");
11893 ContBB->getTerminator()->eraseFromParent();
11894 CurBB->getTerminator()->eraseFromParent();
11895
11896 Builder.CreateCondBr(Cond: SuccessOrFail, True: ExitBB, False: ContBB);
11897
11898 Builder.SetInsertPoint(ContBB);
11899 Builder.CreateStore(Val: OldValue, Ptr: V.Var);
11900 Builder.CreateBr(Dest: ExitBB);
11901
11902 if (UnreachableInst *ExitTI =
11903 dyn_cast<UnreachableInst>(Val: ExitBB->getTerminator())) {
11904 CurBBTI->eraseFromParent();
11905 Builder.SetInsertPoint(ExitBB);
11906 } else {
11907 Builder.SetInsertPoint(ExitTI);
11908 }
11909 } else {
11910 Value *CapturedValue =
11911 Builder.CreateSelect(C: SuccessOrFail, True: E, False: OldValue);
11912 Builder.CreateStore(Val: CapturedValue, Ptr: V.Var, isVolatile: V.IsVolatile);
11913 }
11914 }
11915 }
11916 // The comparison result has to be stored.
11917 if (R.Var) {
11918 assert(R.Var->getType()->isPointerTy() &&
11919 "r.var must be of pointer type");
11920 assert(R.ElemTy->isIntegerTy() && "r must be of integral type");
11921
11922 Value *SuccessFailureVal =
11923 Builder.CreateExtractValue(Agg: Result, /*Idxs=*/1);
11924 Value *ResultCast =
11925 R.IsSigned ? Builder.CreateSExt(V: SuccessFailureVal, DestTy: R.ElemTy)
11926 : Builder.CreateZExt(V: SuccessFailureVal, DestTy: R.ElemTy);
11927 Builder.CreateStore(Val: ResultCast, Ptr: R.Var, isVolatile: R.IsVolatile);
11928 }
11929 }
11930
11931 // For the HandleFPNegZero path, handle V.Var and R.Var using the
11932 // pre-computed OldValue and SuccessOrFail.
11933 if (HandleFPNegZero && !IsInteger) {
11934 if (V.Var) {
11935 assert(OldValue->getType() == V.ElemTy &&
11936 "OldValue and V must be of same type");
11937 if (IsPostfixUpdate) {
11938 Builder.CreateStore(Val: OldValue, Ptr: V.Var, isVolatile: V.IsVolatile);
11939 } else {
11940 if (IsFailOnly) {
11941 BasicBlock *CurBB = Builder.GetInsertBlock();
11942 Instruction *CurBBTI = CurBB->getTerminatorOrNull();
11943 CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable();
11944 BasicBlock *ExitBB = CurBB->splitBasicBlock(
11945 I: CurBBTI, BBName: X.Var->getName() + ".atomic.exit");
11946 BasicBlock *ContBB = CurBB->splitBasicBlock(
11947 I: CurBB->getTerminator(), BBName: X.Var->getName() + ".atomic.cont");
11948 ContBB->getTerminator()->eraseFromParent();
11949 CurBB->getTerminator()->eraseFromParent();
11950
11951 Builder.CreateCondBr(Cond: SuccessOrFail, True: ExitBB, False: ContBB);
11952
11953 Builder.SetInsertPoint(ContBB);
11954 Builder.CreateStore(Val: OldValue, Ptr: V.Var);
11955 Builder.CreateBr(Dest: ExitBB);
11956
11957 if (UnreachableInst *ExitTI =
11958 dyn_cast<UnreachableInst>(Val: ExitBB->getTerminator())) {
11959 CurBBTI->eraseFromParent();
11960 Builder.SetInsertPoint(ExitBB);
11961 } else {
11962 Builder.SetInsertPoint(ExitTI);
11963 }
11964 } else {
11965 Value *CapturedValue =
11966 Builder.CreateSelect(C: SuccessOrFail, True: E, False: OldValue);
11967 Builder.CreateStore(Val: CapturedValue, Ptr: V.Var, isVolatile: V.IsVolatile);
11968 }
11969 }
11970 }
11971 // The comparison result has to be stored.
11972 if (R.Var) {
11973 assert(R.Var->getType()->isPointerTy() &&
11974 "r.var must be of pointer type");
11975 assert(R.ElemTy->isIntegerTy() && "r must be of integral type");
11976
11977 Value *ResultCast = R.IsSigned
11978 ? Builder.CreateSExt(V: SuccessOrFail, DestTy: R.ElemTy)
11979 : Builder.CreateZExt(V: SuccessOrFail, DestTy: R.ElemTy);
11980 Builder.CreateStore(Val: ResultCast, Ptr: R.Var, isVolatile: R.IsVolatile);
11981 }
11982 }
11983 } else {
11984 assert((Op == OMPAtomicCompareOp::MAX || Op == OMPAtomicCompareOp::MIN) &&
11985 "Op should be either max or min at this point");
11986 assert(!IsFailOnly && "IsFailOnly is only valid when the comparison is ==");
11987
11988 // Reverse the ordop as the OpenMP forms are different from LLVM forms.
11989 // Let's take max as example.
11990 // OpenMP form:
11991 // x = x > expr ? expr : x;
11992 // LLVM form:
11993 // *ptr = *ptr > val ? *ptr : val;
11994 // We need to transform to LLVM form.
11995 // x = x <= expr ? x : expr;
11996 AtomicRMWInst::BinOp NewOp;
11997 if (IsXBinopExpr) {
11998 if (IsInteger) {
11999 if (X.IsSigned)
12000 NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::Min
12001 : AtomicRMWInst::Max;
12002 else
12003 NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::UMin
12004 : AtomicRMWInst::UMax;
12005 } else {
12006 NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::FMin
12007 : AtomicRMWInst::FMax;
12008 }
12009 } else {
12010 if (IsInteger) {
12011 if (X.IsSigned)
12012 NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::Max
12013 : AtomicRMWInst::Min;
12014 else
12015 NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::UMax
12016 : AtomicRMWInst::UMin;
12017 } else {
12018 NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::FMax
12019 : AtomicRMWInst::FMin;
12020 }
12021 }
12022
12023 AtomicRMWInst *OldValue =
12024 Builder.CreateAtomicRMW(Op: NewOp, Ptr: X.Var, Val: E, Align: MaybeAlign(), Ordering: AO);
12025 if (V.Var) {
12026 Value *CapturedValue = nullptr;
12027 if (IsPostfixUpdate) {
12028 CapturedValue = OldValue;
12029 } else {
12030 CmpInst::Predicate Pred;
12031 switch (NewOp) {
12032 case AtomicRMWInst::Max:
12033 Pred = CmpInst::ICMP_SGT;
12034 break;
12035 case AtomicRMWInst::UMax:
12036 Pred = CmpInst::ICMP_UGT;
12037 break;
12038 case AtomicRMWInst::FMax:
12039 Pred = CmpInst::FCMP_OGT;
12040 break;
12041 case AtomicRMWInst::Min:
12042 Pred = CmpInst::ICMP_SLT;
12043 break;
12044 case AtomicRMWInst::UMin:
12045 Pred = CmpInst::ICMP_ULT;
12046 break;
12047 case AtomicRMWInst::FMin:
12048 Pred = CmpInst::FCMP_OLT;
12049 break;
12050 default:
12051 llvm_unreachable("unexpected comparison op");
12052 }
12053 Value *NonAtomicCmp = Builder.CreateCmp(Pred, LHS: OldValue, RHS: E);
12054 CapturedValue = Builder.CreateSelect(C: NonAtomicCmp, True: E, False: OldValue);
12055 }
12056 Builder.CreateStore(Val: CapturedValue, Ptr: V.Var, isVolatile: V.IsVolatile);
12057 }
12058 }
12059
12060 checkAndEmitFlushAfterAtomic(Loc, AO, AK: AtomicKind::Compare);
12061
12062 return Builder.saveIP();
12063}
12064
12065OpenMPIRBuilder::InsertPointOrErrorTy
12066OpenMPIRBuilder::createTeams(const LocationDescription &Loc,
12067 BodyGenCallbackTy BodyGenCB, Value *NumTeamsLower,
12068 Value *NumTeamsUpper, Value *ThreadLimit,
12069 Value *IfExpr) {
12070 if (!updateToLocation(Loc))
12071 return InsertPointTy();
12072
12073 uint32_t SrcLocStrSize;
12074 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
12075 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
12076 Function *CurrentFunction = Builder.GetInsertBlock()->getParent();
12077
12078 // Outer allocation basicblock is the entry block of the current function.
12079 BasicBlock &OuterAllocaBB = CurrentFunction->getEntryBlock();
12080 if (&OuterAllocaBB == Builder.GetInsertBlock()) {
12081 BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, Name: "teams.entry");
12082 Builder.SetInsertPoint(TheBB: BodyBB, IP: BodyBB->begin());
12083 }
12084
12085 // The current basic block is split into four basic blocks. After outlining,
12086 // they will be mapped as follows:
12087 // ```
12088 // def current_fn() {
12089 // current_basic_block:
12090 // br label %teams.exit
12091 // teams.exit:
12092 // ; instructions after teams
12093 // }
12094 //
12095 // def outlined_fn() {
12096 // teams.alloca:
12097 // br label %teams.body
12098 // teams.body:
12099 // ; instructions within teams body
12100 // }
12101 // ```
12102 BasicBlock *ExitBB = splitBB(Builder, /*CreateBranch=*/true, Name: "teams.exit");
12103 BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, Name: "teams.body");
12104 BasicBlock *AllocaBB =
12105 splitBB(Builder, /*CreateBranch=*/true, Name: "teams.alloca");
12106
12107 bool SubClausesPresent =
12108 (NumTeamsLower || NumTeamsUpper || ThreadLimit || IfExpr);
12109 // Push num_teams
12110 if (!Config.isTargetDevice() && SubClausesPresent) {
12111 assert((NumTeamsLower == nullptr || NumTeamsUpper != nullptr) &&
12112 "if lowerbound is non-null, then upperbound must also be non-null "
12113 "for bounds on num_teams");
12114
12115 if (NumTeamsUpper == nullptr)
12116 NumTeamsUpper = Builder.getInt32(C: 0);
12117
12118 if (NumTeamsLower == nullptr)
12119 NumTeamsLower = NumTeamsUpper;
12120
12121 if (IfExpr) {
12122 assert(IfExpr->getType()->isIntegerTy() &&
12123 "argument to if clause must be an integer value");
12124
12125 // upper = ifexpr ? upper : 1
12126 if (IfExpr->getType() != Int1)
12127 IfExpr = Builder.CreateICmpNE(LHS: IfExpr,
12128 RHS: ConstantInt::get(Ty: IfExpr->getType(), V: 0));
12129 NumTeamsUpper = Builder.CreateSelect(
12130 C: IfExpr, True: NumTeamsUpper, False: Builder.getInt32(C: 1), Name: "numTeamsUpper");
12131
12132 // lower = ifexpr ? lower : 1
12133 NumTeamsLower = Builder.CreateSelect(
12134 C: IfExpr, True: NumTeamsLower, False: Builder.getInt32(C: 1), Name: "numTeamsLower");
12135 }
12136
12137 if (ThreadLimit == nullptr)
12138 ThreadLimit = Builder.getInt32(C: 0);
12139
12140 // The __kmpc_push_num_teams_51 function expects int32 as the arguments. So,
12141 // truncate or sign extend the passed values to match the int32 parameters.
12142 Value *NumTeamsLowerInt32 =
12143 Builder.CreateSExtOrTrunc(V: NumTeamsLower, DestTy: Builder.getInt32Ty());
12144 Value *NumTeamsUpperInt32 =
12145 Builder.CreateSExtOrTrunc(V: NumTeamsUpper, DestTy: Builder.getInt32Ty());
12146 Value *ThreadLimitInt32 =
12147 Builder.CreateSExtOrTrunc(V: ThreadLimit, DestTy: Builder.getInt32Ty());
12148
12149 Value *ThreadNum = getOrCreateThreadID(Ident);
12150
12151 createRuntimeFunctionCall(
12152 Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_push_num_teams_51),
12153 Args: {Ident, ThreadNum, NumTeamsLowerInt32, NumTeamsUpperInt32,
12154 ThreadLimitInt32});
12155 }
12156 // Generate the body of teams.
12157 InsertPointTy AllocaIP(AllocaBB, AllocaBB->begin());
12158 InsertPointTy CodeGenIP(BodyBB, BodyBB->begin());
12159 if (Error Err = BodyGenCB(AllocaIP, CodeGenIP, ExitBB))
12160 return Err;
12161
12162 auto OI = std::make_unique<OutlineInfo>();
12163 OI->EntryBB = AllocaBB;
12164 OI->ExitBB = ExitBB;
12165 OI->OuterAllocBB = &OuterAllocaBB;
12166
12167 // Insert fake values for global tid and bound tid.
12168 SmallVector<Instruction *, 8> ToBeDeleted;
12169 InsertPointTy OuterAllocaIP(&OuterAllocaBB, OuterAllocaBB.begin());
12170 OI->ExcludeArgsFromAggregate.push_back(Elt: createFakeIntVal(
12171 Builder, OuterAllocaIP, ToBeDeleted, InnerAllocaIP: AllocaIP, Name: "gid", AsPtr: true));
12172 OI->ExcludeArgsFromAggregate.push_back(Elt: createFakeIntVal(
12173 Builder, OuterAllocaIP, ToBeDeleted, InnerAllocaIP: AllocaIP, Name: "tid", AsPtr: true));
12174
12175 auto HostPostOutlineCB = [this, Ident,
12176 ToBeDeleted](Function &OutlinedFn) mutable {
12177 // The stale call instruction will be replaced with a new call instruction
12178 // for runtime call with the outlined function.
12179
12180 assert(OutlinedFn.hasOneUse() &&
12181 "there must be a single user for the outlined function");
12182 CallInst *StaleCI = cast<CallInst>(Val: OutlinedFn.user_back());
12183 ToBeDeleted.push_back(Elt: StaleCI);
12184
12185 assert((OutlinedFn.arg_size() == 2 || OutlinedFn.arg_size() == 3) &&
12186 "Outlined function must have two or three arguments only");
12187
12188 bool HasShared = OutlinedFn.arg_size() == 3;
12189
12190 OutlinedFn.getArg(i: 0)->setName("global.tid.ptr");
12191 OutlinedFn.getArg(i: 1)->setName("bound.tid.ptr");
12192 if (HasShared)
12193 OutlinedFn.getArg(i: 2)->setName("data");
12194
12195 // Call to the runtime function for teams in the current function.
12196 assert(StaleCI && "Error while outlining - no CallInst user found for the "
12197 "outlined function.");
12198 Builder.SetInsertPoint(StaleCI);
12199 SmallVector<Value *> Args = {
12200 Ident, Builder.getInt32(C: StaleCI->arg_size() - 2), &OutlinedFn};
12201 if (HasShared)
12202 Args.push_back(Elt: StaleCI->getArgOperand(i: 2));
12203 createRuntimeFunctionCall(
12204 Callee: getOrCreateRuntimeFunctionPtr(
12205 FnID: omp::RuntimeFunction::OMPRTL___kmpc_fork_teams),
12206 Args);
12207
12208 Builder.ClearInsertionPoint();
12209 for (Instruction *I : llvm::reverse(C&: ToBeDeleted))
12210 I->eraseFromParent();
12211 };
12212
12213 if (!Config.isTargetDevice())
12214 OI->PostOutlineCB = HostPostOutlineCB;
12215
12216 addOutlineInfo(OI: std::move(OI));
12217
12218 Builder.SetInsertPoint(ExitBB);
12219
12220 return Builder.saveIP();
12221}
12222
12223OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createDistribute(
12224 const LocationDescription &Loc, InsertPointTy OuterAllocIP,
12225 ArrayRef<BasicBlock *> OuterDeallocBlocks, BodyGenCallbackTy BodyGenCB) {
12226 if (!updateToLocation(Loc))
12227 return InsertPointTy();
12228
12229 BasicBlock *OuterAllocaBB = OuterAllocIP.getBlock();
12230
12231 if (OuterAllocaBB == Builder.GetInsertBlock()) {
12232 BasicBlock *BodyBB =
12233 splitBB(Builder, /*CreateBranch=*/true, Name: "distribute.entry");
12234 Builder.SetInsertPoint(TheBB: BodyBB, IP: BodyBB->begin());
12235 }
12236 BasicBlock *ExitBB =
12237 splitBB(Builder, /*CreateBranch=*/true, Name: "distribute.exit");
12238 BasicBlock *BodyBB =
12239 splitBB(Builder, /*CreateBranch=*/true, Name: "distribute.body");
12240 BasicBlock *AllocaBB =
12241 splitBB(Builder, /*CreateBranch=*/true, Name: "distribute.alloca");
12242
12243 // Generate the body of distribute clause
12244 InsertPointTy AllocaIP(AllocaBB, AllocaBB->begin());
12245 InsertPointTy CodeGenIP(BodyBB, BodyBB->begin());
12246 if (Error Err = BodyGenCB(AllocaIP, CodeGenIP, ExitBB))
12247 return Err;
12248
12249 // When using target we use different runtime functions which require a
12250 // callback.
12251 if (Config.isTargetDevice()) {
12252 auto OI = std::make_unique<OutlineInfo>();
12253 OI->OuterAllocBB = OuterAllocIP.getBlock();
12254 OI->EntryBB = AllocaBB;
12255 OI->ExitBB = ExitBB;
12256 OI->OuterDeallocBBs.reserve(N: OuterDeallocBlocks.size());
12257 copy(Range&: OuterDeallocBlocks, Out: OI->OuterDeallocBBs.end());
12258
12259 addOutlineInfo(OI: std::move(OI));
12260 }
12261 Builder.SetInsertPoint(ExitBB);
12262
12263 return Builder.saveIP();
12264}
12265
12266GlobalVariable *
12267OpenMPIRBuilder::createOffloadMapnames(SmallVectorImpl<llvm::Constant *> &Names,
12268 std::string VarName) {
12269 llvm::Constant *MapNamesArrayInit = llvm::ConstantArray::get(
12270 T: llvm::ArrayType::get(ElementType: llvm::PointerType::getUnqual(C&: M.getContext()),
12271 NumElements: Names.size()),
12272 V: Names);
12273 auto *MapNamesArrayGlobal = new llvm::GlobalVariable(
12274 M, MapNamesArrayInit->getType(),
12275 /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage, MapNamesArrayInit,
12276 VarName);
12277 return MapNamesArrayGlobal;
12278}
12279
12280// Create all simple and struct types exposed by the runtime and remember
12281// the llvm::PointerTypes of them for easy access later.
12282void OpenMPIRBuilder::initializeTypes(Module &M) {
12283 LLVMContext &Ctx = M.getContext();
12284 StructType *T;
12285 unsigned DefaultTargetAS = Config.getDefaultTargetAS();
12286 unsigned ProgramAS = M.getDataLayout().getProgramAddressSpace();
12287#define OMP_TYPE(VarName, InitValue) VarName = InitValue;
12288#define OMP_ARRAY_TYPE(VarName, ElemTy, ArraySize) \
12289 VarName##Ty = ArrayType::get(ElemTy, ArraySize); \
12290 VarName##PtrTy = PointerType::get(Ctx, DefaultTargetAS);
12291#define OMP_FUNCTION_TYPE(VarName, IsVarArg, ReturnType, ...) \
12292 VarName = FunctionType::get(ReturnType, {__VA_ARGS__}, IsVarArg); \
12293 VarName##Ptr = PointerType::get(Ctx, ProgramAS);
12294#define OMP_STRUCT_TYPE(VarName, StructName, Packed, ...) \
12295 T = StructType::getTypeByName(Ctx, StructName); \
12296 if (!T) \
12297 T = StructType::create(Ctx, {__VA_ARGS__}, StructName, Packed); \
12298 VarName = T; \
12299 VarName##Ptr = PointerType::get(Ctx, DefaultTargetAS);
12300#include "llvm/Frontend/OpenMP/OMPKinds.def"
12301}
12302
12303void OpenMPIRBuilder::OutlineInfo::collectBlocks(
12304 SmallPtrSetImpl<BasicBlock *> &BlockSet,
12305 SmallVectorImpl<BasicBlock *> &BlockVector) {
12306 SmallVector<BasicBlock *, 32> Worklist;
12307 BlockSet.insert(Ptr: EntryBB);
12308 BlockSet.insert(Ptr: ExitBB);
12309
12310 Worklist.push_back(Elt: EntryBB);
12311 while (!Worklist.empty()) {
12312 BasicBlock *BB = Worklist.pop_back_val();
12313 BlockVector.push_back(Elt: BB);
12314 for (BasicBlock *SuccBB : successors(BB))
12315 if (BlockSet.insert(Ptr: SuccBB).second)
12316 Worklist.push_back(Elt: SuccBB);
12317 }
12318}
12319
12320std::unique_ptr<CodeExtractor>
12321OpenMPIRBuilder::OutlineInfo::createCodeExtractor(ArrayRef<BasicBlock *> Blocks,
12322 bool ArgsInZeroAddressSpace,
12323 Twine Suffix) {
12324 return std::make_unique<CodeExtractor>(
12325 args&: Blocks, /* DominatorTree */ args: nullptr,
12326 /* AggregateArgs */ args: true,
12327 /* BlockFrequencyInfo */ args: nullptr,
12328 /* BranchProbabilityInfo */ args: nullptr,
12329 /* AssumptionCache */ args: nullptr,
12330 /* AllowVarArgs */ args: true,
12331 /* AllowAlloca */ args: true,
12332 /* AllocationBlock*/ args&: OuterAllocBB,
12333 /* DeallocationBlocks */ args: ArrayRef<BasicBlock *>(),
12334 /* Suffix */ args: Suffix.str(), args&: ArgsInZeroAddressSpace);
12335}
12336
12337std::unique_ptr<CodeExtractor> DeviceSharedMemOutlineInfo::createCodeExtractor(
12338 ArrayRef<BasicBlock *> Blocks, bool ArgsInZeroAddressSpace, Twine Suffix) {
12339 return std::make_unique<DeviceSharedMemCodeExtractor>(
12340 args&: OMPBuilder, args&: Blocks, /* DominatorTree */ args: nullptr,
12341 /* AggregateArgs */ args: true,
12342 /* BlockFrequencyInfo */ args: nullptr,
12343 /* BranchProbabilityInfo */ args: nullptr,
12344 /* AssumptionCache */ args: nullptr,
12345 /* AllowVarArgs */ args: true,
12346 /* AllowAlloca */ args: true,
12347 /* AllocationBlock*/ args&: OuterAllocBB,
12348 /* DeallocationBlocks */ args: OuterDeallocBBs.empty()
12349 ? SmallVector<BasicBlock *>{ExitBB}
12350 : OuterDeallocBBs,
12351 /* Suffix */ args: Suffix.str(), args&: ArgsInZeroAddressSpace);
12352}
12353
12354void OpenMPIRBuilder::createOffloadEntry(Constant *ID, Constant *Addr,
12355 uint64_t Size, int32_t Flags,
12356 GlobalValue::LinkageTypes,
12357 StringRef Name) {
12358 if (!Config.isGPU()) {
12359 llvm::offloading::emitOffloadingEntry(
12360 M, Kind: object::OffloadKind::OFK_OpenMP, Addr: ID,
12361 Name: Name.empty() ? Addr->getName() : Name, Size, Flags, /*Data=*/0);
12362 return;
12363 }
12364 // TODO: Add support for global variables on the device after declare target
12365 // support.
12366 Function *Fn = dyn_cast<Function>(Val: Addr);
12367 if (!Fn)
12368 return;
12369
12370 // Add a function attribute for the kernel.
12371 Fn->addFnAttr(Kind: "kernel");
12372 if (T.isAMDGCN())
12373 Fn->addFnAttr(Kind: "uniform-work-group-size");
12374 Fn->addFnAttr(Kind: Attribute::MustProgress);
12375}
12376
12377// We only generate metadata for function that contain target regions.
12378void OpenMPIRBuilder::createOffloadEntriesAndInfoMetadata(
12379 EmitMetadataErrorReportFunctionTy &ErrorFn) {
12380
12381 // If there are no entries, we don't need to do anything.
12382 if (OffloadInfoManager.empty())
12383 return;
12384
12385 LLVMContext &C = M.getContext();
12386 SmallVector<std::pair<const OffloadEntriesInfoManager::OffloadEntryInfo *,
12387 TargetRegionEntryInfo>,
12388 16>
12389 OrderedEntries(OffloadInfoManager.size());
12390
12391 // Auxiliary methods to create metadata values and strings.
12392 auto &&GetMDInt = [this](unsigned V) {
12393 return ConstantAsMetadata::get(C: ConstantInt::get(Ty: Builder.getInt32Ty(), V));
12394 };
12395
12396 auto &&GetMDString = [&C](StringRef V) { return MDString::get(Context&: C, Str: V); };
12397
12398 // Create the offloading info metadata node.
12399 NamedMDNode *MD = M.getOrInsertNamedMetadata(Name: "omp_offload.info");
12400 auto &&TargetRegionMetadataEmitter =
12401 [&C, MD, &OrderedEntries, &GetMDInt, &GetMDString](
12402 const TargetRegionEntryInfo &EntryInfo,
12403 const OffloadEntriesInfoManager::OffloadEntryInfoTargetRegion &E) {
12404 // Generate metadata for target regions. Each entry of this metadata
12405 // contains:
12406 // - Entry 0 -> Kind of this type of metadata (0).
12407 // - Entry 1 -> Device ID of the file where the entry was identified.
12408 // - Entry 2 -> File ID of the file where the entry was identified.
12409 // - Entry 3 -> Mangled name of the function where the entry was
12410 // identified.
12411 // - Entry 4 -> Line in the file where the entry was identified.
12412 // - Entry 5 -> Count of regions at this DeviceID/FilesID/Line.
12413 // - Entry 6 -> Order the entry was created.
12414 // The first element of the metadata node is the kind.
12415 Metadata *Ops[] = {
12416 GetMDInt(E.getKind()), GetMDInt(EntryInfo.DeviceID),
12417 GetMDInt(EntryInfo.FileID), GetMDString(EntryInfo.ParentName),
12418 GetMDInt(EntryInfo.Line), GetMDInt(EntryInfo.Count),
12419 GetMDInt(E.getOrder())};
12420
12421 // Save this entry in the right position of the ordered entries array.
12422 OrderedEntries[E.getOrder()] = std::make_pair(x: &E, y: EntryInfo);
12423
12424 // Add metadata to the named metadata node.
12425 MD->addOperand(M: MDNode::get(Context&: C, MDs: Ops));
12426 };
12427
12428 OffloadInfoManager.actOnTargetRegionEntriesInfo(Action: TargetRegionMetadataEmitter);
12429
12430 // Create function that emits metadata for each device global variable entry;
12431 auto &&DeviceGlobalVarMetadataEmitter =
12432 [&C, &OrderedEntries, &GetMDInt, &GetMDString, MD](
12433 StringRef MangledName,
12434 const OffloadEntriesInfoManager::OffloadEntryInfoDeviceGlobalVar &E) {
12435 // Generate metadata for global variables. Each entry of this metadata
12436 // contains:
12437 // - Entry 0 -> Kind of this type of metadata (1).
12438 // - Entry 1 -> Mangled name of the variable.
12439 // - Entry 2 -> Declare target kind.
12440 // - Entry 3 -> Order the entry was created.
12441 // The first element of the metadata node is the kind.
12442 Metadata *Ops[] = {GetMDInt(E.getKind()), GetMDString(MangledName),
12443 GetMDInt(E.getFlags()), GetMDInt(E.getOrder())};
12444
12445 // Save this entry in the right position of the ordered entries array.
12446 TargetRegionEntryInfo varInfo(MangledName, 0, 0, 0);
12447 OrderedEntries[E.getOrder()] = std::make_pair(x: &E, y&: varInfo);
12448
12449 // Add metadata to the named metadata node.
12450 MD->addOperand(M: MDNode::get(Context&: C, MDs: Ops));
12451 };
12452
12453 OffloadInfoManager.actOnDeviceGlobalVarEntriesInfo(
12454 Action: DeviceGlobalVarMetadataEmitter);
12455
12456 for (const auto &E : OrderedEntries) {
12457 assert(E.first && "All ordered entries must exist!");
12458 if (const auto *CE =
12459 dyn_cast<OffloadEntriesInfoManager::OffloadEntryInfoTargetRegion>(
12460 Val: E.first)) {
12461 if (!CE->getID() || !CE->getAddress()) {
12462 // Do not blame the entry if the parent funtion is not emitted.
12463 TargetRegionEntryInfo EntryInfo = E.second;
12464 StringRef FnName = EntryInfo.ParentName;
12465 if (!M.getNamedValue(Name: FnName))
12466 continue;
12467 ErrorFn(EMIT_MD_TARGET_REGION_ERROR, EntryInfo);
12468 continue;
12469 }
12470 createOffloadEntry(ID: CE->getID(), Addr: CE->getAddress(),
12471 /*Size=*/0, Flags: CE->getFlags(),
12472 GlobalValue::WeakAnyLinkage);
12473 } else if (const auto *CE = dyn_cast<
12474 OffloadEntriesInfoManager::OffloadEntryInfoDeviceGlobalVar>(
12475 Val: E.first)) {
12476 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind Flags =
12477 static_cast<OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind>(
12478 CE->getFlags());
12479 switch (Flags) {
12480 case OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter:
12481 case OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo:
12482 if (Config.isTargetDevice() && Config.hasRequiresUnifiedSharedMemory())
12483 continue;
12484 if (!CE->getAddress()) {
12485 ErrorFn(EMIT_MD_DECLARE_TARGET_ERROR, E.second);
12486 continue;
12487 }
12488 // The vaiable has no definition - no need to add the entry.
12489 if (CE->getVarSize() == 0)
12490 continue;
12491 break;
12492 case OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink:
12493 assert(((Config.isTargetDevice() && !CE->getAddress()) ||
12494 (!Config.isTargetDevice() && CE->getAddress())) &&
12495 "Declaret target link address is set.");
12496 if (Config.isTargetDevice())
12497 continue;
12498 if (!CE->getAddress()) {
12499 ErrorFn(EMIT_MD_GLOBAL_VAR_LINK_ERROR, TargetRegionEntryInfo());
12500 continue;
12501 }
12502 break;
12503 case OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect:
12504 case OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirectVTable:
12505 if (!CE->getAddress()) {
12506 ErrorFn(EMIT_MD_GLOBAL_VAR_INDIRECT_ERROR, E.second);
12507 continue;
12508 }
12509 break;
12510 default:
12511 break;
12512 }
12513
12514 // Hidden or internal symbols on the device are not externally visible.
12515 // We should not attempt to register them by creating an offloading
12516 // entry. Indirect variables are handled separately on the device.
12517 if (auto *GV = dyn_cast<GlobalValue>(Val: CE->getAddress()))
12518 if ((GV->hasLocalLinkage() || GV->hasHiddenVisibility()) &&
12519 (Flags !=
12520 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect &&
12521 Flags != OffloadEntriesInfoManager::
12522 OMPTargetGlobalVarEntryIndirectVTable))
12523 continue;
12524
12525 // Indirect globals need to use a special name that doesn't match the name
12526 // of the associated host global.
12527 if (Flags == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect ||
12528 Flags ==
12529 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirectVTable)
12530 createOffloadEntry(ID: CE->getAddress(), Addr: CE->getAddress(), Size: CE->getVarSize(),
12531 Flags, CE->getLinkage(), Name: CE->getVarName());
12532 else
12533 createOffloadEntry(ID: CE->getAddress(), Addr: CE->getAddress(), Size: CE->getVarSize(),
12534 Flags, CE->getLinkage());
12535
12536 } else {
12537 llvm_unreachable("Unsupported entry kind.");
12538 }
12539 }
12540
12541 // Emit requires directive globals to a special entry so the runtime can
12542 // register them when the device image is loaded.
12543 // TODO: This reduces the offloading entries to a 32-bit integer. Offloading
12544 // entries should be redesigned to better suit this use-case.
12545 if (Config.hasRequiresFlags() && !Config.isTargetDevice())
12546 offloading::emitOffloadingEntry(
12547 M, Kind: object::OffloadKind::OFK_OpenMP,
12548 Addr: Constant::getNullValue(Ty: PointerType::getUnqual(C&: M.getContext())),
12549 Name: ".requires", /*Size=*/0,
12550 Flags: OffloadEntriesInfoManager::OMPTargetGlobalRegisterRequires,
12551 Data: Config.getRequiresFlags());
12552}
12553
12554void TargetRegionEntryInfo::getTargetRegionEntryFnName(
12555 SmallVectorImpl<char> &Name, StringRef ParentName, unsigned DeviceID,
12556 unsigned FileID, unsigned Line, unsigned Count) {
12557 raw_svector_ostream OS(Name);
12558 OS << KernelNamePrefix << llvm::format(Fmt: "%x", Vals: DeviceID)
12559 << llvm::format(Fmt: "_%x_", Vals: FileID) << ParentName << "_l" << Line;
12560 if (Count)
12561 OS << "_" << Count;
12562}
12563
12564void OffloadEntriesInfoManager::getTargetRegionEntryFnName(
12565 SmallVectorImpl<char> &Name, const TargetRegionEntryInfo &EntryInfo) {
12566 unsigned NewCount = getTargetRegionEntryInfoCount(EntryInfo);
12567 TargetRegionEntryInfo::getTargetRegionEntryFnName(
12568 Name, ParentName: EntryInfo.ParentName, DeviceID: EntryInfo.DeviceID, FileID: EntryInfo.FileID,
12569 Line: EntryInfo.Line, Count: NewCount);
12570}
12571
12572TargetRegionEntryInfo
12573OpenMPIRBuilder::getTargetEntryUniqueInfo(FileIdentifierInfoCallbackTy CallBack,
12574 vfs::FileSystem &VFS,
12575 StringRef ParentName) {
12576 sys::fs::UniqueID ID(0xdeadf17e, 0);
12577 auto FileIDInfo = CallBack();
12578 uint64_t FileID = 0;
12579 if (ErrorOr<vfs::Status> Status = VFS.status(Path: std::get<0>(t&: FileIDInfo))) {
12580 ID = Status->getUniqueID();
12581 FileID = Status->getUniqueID().getFile();
12582 } else {
12583 // If the inode ID could not be determined, create a hash value
12584 // the current file name and use that as an ID.
12585 FileID = hash_value(arg: std::get<0>(t&: FileIDInfo));
12586 }
12587
12588 return TargetRegionEntryInfo(ParentName, ID.getDevice(), FileID,
12589 std::get<1>(t&: FileIDInfo));
12590}
12591
12592unsigned OpenMPIRBuilder::getFlagMemberOffset() {
12593 unsigned Offset = 0;
12594 for (uint64_t Remain =
12595 static_cast<std::underlying_type_t<omp::OpenMPOffloadMappingFlags>>(
12596 omp::OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);
12597 !(Remain & 1); Remain = Remain >> 1)
12598 Offset++;
12599 return Offset;
12600}
12601
12602omp::OpenMPOffloadMappingFlags
12603OpenMPIRBuilder::getMemberOfFlag(unsigned Position) {
12604 // Rotate by getFlagMemberOffset() bits.
12605 return static_cast<omp::OpenMPOffloadMappingFlags>(((uint64_t)Position + 1)
12606 << getFlagMemberOffset());
12607}
12608
12609void OpenMPIRBuilder::setCorrectMemberOfFlag(
12610 omp::OpenMPOffloadMappingFlags &Flags,
12611 omp::OpenMPOffloadMappingFlags MemberOfFlag) {
12612 // If the entry is PTR_AND_OBJ but has not been marked with the special
12613 // placeholder value 0xFFFF in the MEMBER_OF field, then it should not be
12614 // marked as MEMBER_OF.
12615 if (static_cast<std::underlying_type_t<omp::OpenMPOffloadMappingFlags>>(
12616 Flags & omp::OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ) &&
12617 static_cast<std::underlying_type_t<omp::OpenMPOffloadMappingFlags>>(
12618 (Flags & omp::OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) !=
12619 omp::OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF))
12620 return;
12621
12622 // Entries with ATTACH are not members-of anything. They are handled
12623 // separately by the runtime after other maps have been handled.
12624 if (static_cast<std::underlying_type_t<omp::OpenMPOffloadMappingFlags>>(
12625 Flags & omp::OpenMPOffloadMappingFlags::OMP_MAP_ATTACH))
12626 return;
12627
12628 // Reset the placeholder value to prepare the flag for the assignment of the
12629 // proper MEMBER_OF value.
12630 Flags &= ~omp::OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF;
12631 Flags |= MemberOfFlag;
12632}
12633
12634Constant *OpenMPIRBuilder::getAddrOfDeclareTargetVar(
12635 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind CaptureClause,
12636 OffloadEntriesInfoManager::OMPTargetDeviceClauseKind DeviceClause,
12637 bool IsDeclaration, bool IsExternallyVisible,
12638 TargetRegionEntryInfo EntryInfo, StringRef MangledName,
12639 std::vector<GlobalVariable *> &GeneratedRefs, bool OpenMPSIMD,
12640 std::vector<Triple> TargetTriple, Type *LlvmPtrTy,
12641 std::function<Constant *()> GlobalInitializer,
12642 std::function<GlobalValue::LinkageTypes()> VariableLinkage) {
12643 // TODO: convert this to utilise the IRBuilder Config rather than
12644 // a passed down argument.
12645 if (OpenMPSIMD)
12646 return nullptr;
12647
12648 if (CaptureClause == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink ||
12649 ((CaptureClause == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo ||
12650 CaptureClause ==
12651 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter) &&
12652 Config.hasRequiresUnifiedSharedMemory())) {
12653 SmallString<64> PtrName;
12654 {
12655 raw_svector_ostream OS(PtrName);
12656 OS << MangledName;
12657 if (!IsExternallyVisible)
12658 OS << format(Fmt: "_%x", Vals: EntryInfo.FileID);
12659 OS << "_decl_tgt_ref_ptr";
12660 }
12661
12662 Value *Ptr = M.getNamedValue(Name: PtrName);
12663
12664 if (!Ptr) {
12665 GlobalValue *GlobalValue = M.getNamedValue(Name: MangledName);
12666 Ptr = getOrCreateInternalVariable(Ty: LlvmPtrTy, Name: PtrName);
12667
12668 auto *GV = cast<GlobalVariable>(Val: Ptr);
12669 GV->setLinkage(GlobalValue::WeakAnyLinkage);
12670
12671 if (!Config.isTargetDevice()) {
12672 if (GlobalInitializer)
12673 GV->setInitializer(GlobalInitializer());
12674 else
12675 GV->setInitializer(GlobalValue);
12676 }
12677
12678 registerTargetGlobalVariable(
12679 CaptureClause, DeviceClause, IsDeclaration, IsExternallyVisible,
12680 EntryInfo, MangledName, GeneratedRefs, OpenMPSIMD, TargetTriple,
12681 GlobalInitializer, VariableLinkage, LlvmPtrTy, Addr: cast<Constant>(Val: Ptr));
12682 }
12683
12684 return cast<Constant>(Val: Ptr);
12685 }
12686
12687 return nullptr;
12688}
12689
12690void OpenMPIRBuilder::registerTargetGlobalVariable(
12691 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind CaptureClause,
12692 OffloadEntriesInfoManager::OMPTargetDeviceClauseKind DeviceClause,
12693 bool IsDeclaration, bool IsExternallyVisible,
12694 TargetRegionEntryInfo EntryInfo, StringRef MangledName,
12695 std::vector<GlobalVariable *> &GeneratedRefs, bool OpenMPSIMD,
12696 std::vector<Triple> TargetTriple,
12697 std::function<Constant *()> GlobalInitializer,
12698 std::function<GlobalValue::LinkageTypes()> VariableLinkage, Type *LlvmPtrTy,
12699 Constant *Addr) {
12700 if (DeviceClause != OffloadEntriesInfoManager::OMPTargetDeviceClauseAny ||
12701 (TargetTriple.empty() && !Config.isTargetDevice()))
12702 return;
12703
12704 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind Flags;
12705 StringRef VarName;
12706 int64_t VarSize;
12707 GlobalValue::LinkageTypes Linkage;
12708
12709 if ((CaptureClause == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo ||
12710 CaptureClause ==
12711 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter) &&
12712 !Config.hasRequiresUnifiedSharedMemory()) {
12713 Flags = OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo;
12714 VarName = MangledName;
12715 GlobalValue *LlvmVal = M.getNamedValue(Name: VarName);
12716
12717 if (!IsDeclaration)
12718 VarSize = divideCeil(
12719 Numerator: M.getDataLayout().getTypeSizeInBits(Ty: LlvmVal->getValueType()), Denominator: 8);
12720 else
12721 VarSize = 0;
12722 Linkage = (VariableLinkage) ? VariableLinkage() : LlvmVal->getLinkage();
12723
12724 // This is a workaround carried over from Clang which prevents undesired
12725 // optimisation of internal variables.
12726 if (Config.isTargetDevice() &&
12727 (!IsExternallyVisible || Linkage == GlobalValue::LinkOnceODRLinkage)) {
12728 // Do not create a "ref-variable" if the original is not also available
12729 // on the host.
12730 if (!OffloadInfoManager.hasDeviceGlobalVarEntryInfo(VarName))
12731 return;
12732
12733 std::string RefName = createPlatformSpecificName(Parts: {VarName, "ref"});
12734
12735 if (!M.getNamedValue(Name: RefName)) {
12736 Constant *AddrRef =
12737 getOrCreateInternalVariable(Ty: Addr->getType(), Name: RefName);
12738 auto *GvAddrRef = cast<GlobalVariable>(Val: AddrRef);
12739 GvAddrRef->setConstant(true);
12740 GvAddrRef->setLinkage(GlobalValue::InternalLinkage);
12741 GvAddrRef->setInitializer(Addr);
12742 GeneratedRefs.push_back(x: GvAddrRef);
12743 }
12744 }
12745 } else {
12746 if (CaptureClause == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink)
12747 Flags = OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink;
12748 else
12749 Flags = OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo;
12750
12751 if (Config.isTargetDevice()) {
12752 VarName = (Addr) ? Addr->getName() : "";
12753 Addr = nullptr;
12754 } else {
12755 Addr = getAddrOfDeclareTargetVar(
12756 CaptureClause, DeviceClause, IsDeclaration, IsExternallyVisible,
12757 EntryInfo, MangledName, GeneratedRefs, OpenMPSIMD, TargetTriple,
12758 LlvmPtrTy, GlobalInitializer, VariableLinkage);
12759 VarName = (Addr) ? Addr->getName() : "";
12760 }
12761 VarSize = M.getDataLayout().getPointerSize();
12762 Linkage = GlobalValue::WeakAnyLinkage;
12763 }
12764
12765 OffloadInfoManager.registerDeviceGlobalVarEntryInfo(VarName, Addr, VarSize,
12766 Flags, Linkage);
12767}
12768
12769/// Loads all the offload entries information from the host IR
12770/// metadata.
12771void OpenMPIRBuilder::loadOffloadInfoMetadata(Module &M) {
12772 // If we are in target mode, load the metadata from the host IR. This code has
12773 // to match the metadata creation in createOffloadEntriesAndInfoMetadata().
12774
12775 NamedMDNode *MD = M.getNamedMetadata(Name: ompOffloadInfoName);
12776 if (!MD)
12777 return;
12778
12779 for (MDNode *MN : MD->operands()) {
12780 auto &&GetMDInt = [MN](unsigned Idx) {
12781 auto *V = cast<ConstantAsMetadata>(Val: MN->getOperand(I: Idx));
12782 return cast<ConstantInt>(Val: V->getValue())->getZExtValue();
12783 };
12784
12785 auto &&GetMDString = [MN](unsigned Idx) {
12786 auto *V = cast<MDString>(Val: MN->getOperand(I: Idx));
12787 return V->getString();
12788 };
12789
12790 switch (GetMDInt(0)) {
12791 default:
12792 llvm_unreachable("Unexpected metadata!");
12793 break;
12794 case OffloadEntriesInfoManager::OffloadEntryInfo::
12795 OffloadingEntryInfoTargetRegion: {
12796 TargetRegionEntryInfo EntryInfo(/*ParentName=*/GetMDString(3),
12797 /*DeviceID=*/GetMDInt(1),
12798 /*FileID=*/GetMDInt(2),
12799 /*Line=*/GetMDInt(4),
12800 /*Count=*/GetMDInt(5));
12801 OffloadInfoManager.initializeTargetRegionEntryInfo(EntryInfo,
12802 /*Order=*/GetMDInt(6));
12803 break;
12804 }
12805 case OffloadEntriesInfoManager::OffloadEntryInfo::
12806 OffloadingEntryInfoDeviceGlobalVar:
12807 OffloadInfoManager.initializeDeviceGlobalVarEntryInfo(
12808 /*MangledName=*/Name: GetMDString(1),
12809 Flags: static_cast<OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind>(
12810 /*Flags=*/GetMDInt(2)),
12811 /*Order=*/GetMDInt(3));
12812 break;
12813 }
12814 }
12815}
12816
12817void OpenMPIRBuilder::loadOffloadInfoMetadata(vfs::FileSystem &VFS,
12818 StringRef HostFilePath) {
12819 if (HostFilePath.empty())
12820 return;
12821
12822 auto Buf = VFS.getBufferForFile(Name: HostFilePath);
12823 if (std::error_code Err = Buf.getError()) {
12824 report_fatal_error(reason: ("error opening host file from host file path inside of "
12825 "OpenMPIRBuilder: " +
12826 Err.message())
12827 .c_str());
12828 }
12829
12830 LLVMContext Ctx;
12831 auto M = expectedToErrorOrAndEmitErrors(
12832 Ctx, Val: parseBitcodeFile(Buffer: Buf.get()->getMemBufferRef(), Context&: Ctx));
12833 if (std::error_code Err = M.getError()) {
12834 report_fatal_error(
12835 reason: ("error parsing host file inside of OpenMPIRBuilder: " + Err.message())
12836 .c_str());
12837 }
12838
12839 loadOffloadInfoMetadata(M&: *M.get());
12840}
12841
12842OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createIteratorLoop(
12843 LocationDescription Loc, llvm::Value *TripCount, IteratorBodyGenTy BodyGen,
12844 llvm::StringRef Name) {
12845 Builder.restoreIP(IP: Loc.IP);
12846
12847 BasicBlock *CurBB = Builder.GetInsertBlock();
12848 assert(CurBB &&
12849 "expected a valid insertion block for creating an iterator loop");
12850 Function *F = CurBB->getParent();
12851
12852 InsertPointTy SplitIP = Builder.saveIP();
12853 if (SplitIP.getPoint() == CurBB->end())
12854 if (Instruction *Terminator = CurBB->getTerminatorOrNull())
12855 SplitIP = InsertPointTy(CurBB, Terminator->getIterator());
12856
12857 BasicBlock *ContBB =
12858 splitBB(IP: SplitIP, /*CreateBranch=*/false,
12859 DL: Builder.getCurrentDebugLocation(), Name: "omp.it.cont");
12860
12861 CanonicalLoopInfo *CLI =
12862 createLoopSkeleton(DL: Builder.getCurrentDebugLocation(), TripCount, F,
12863 /*PreInsertBefore=*/ContBB,
12864 /*PostInsertBefore=*/ContBB, Name);
12865
12866 // Enter loop from original block.
12867 redirectTo(Source: CurBB, Target: CLI->getPreheader(), DL: Builder.getCurrentDebugLocation());
12868
12869 // Remove the unconditional branch inserted by createLoopSkeleton in the body
12870 if (Instruction *T = CLI->getBody()->getTerminatorOrNull())
12871 T->eraseFromParent();
12872
12873 InsertPointTy BodyIP = CLI->getBodyIP();
12874 if (llvm::Error Err = BodyGen(BodyIP, CLI->getIndVar()))
12875 return Err;
12876
12877 // Body must either fallthrough to the latch or branch directly to it.
12878 if (Instruction *BodyTerminator = CLI->getBody()->getTerminatorOrNull()) {
12879 auto *BodyBr = dyn_cast<UncondBrInst>(Val: BodyTerminator);
12880 if (!BodyBr || BodyBr->getSuccessor() != CLI->getLatch()) {
12881 return make_error<StringError>(
12882 Args: "iterator bodygen must terminate the canonical body with an "
12883 "unconditional branch to the loop latch",
12884 Args: inconvertibleErrorCode());
12885 }
12886 } else {
12887 // Ensure we end the loop body by jumping to the latch.
12888 Builder.SetInsertPoint(CLI->getBody());
12889 Builder.CreateBr(Dest: CLI->getLatch());
12890 }
12891
12892 // Link After -> ContBB
12893 Builder.SetInsertPoint(TheBB: CLI->getAfter(), IP: CLI->getAfter()->begin());
12894 if (!CLI->getAfter()->hasTerminator())
12895 Builder.CreateBr(Dest: ContBB);
12896
12897 return InsertPointTy{ContBB, ContBB->begin()};
12898}
12899
12900/// Mangle the parameter part of the vector function name according to
12901/// their OpenMP classification. The mangling function is defined in
12902/// section 4.5 of the AAVFABI(2021Q1).
12903static std::string mangleVectorParameters(
12904 ArrayRef<llvm::OpenMPIRBuilder::DeclareSimdAttrTy> ParamAttrs) {
12905 SmallString<256> Buffer;
12906 llvm::raw_svector_ostream Out(Buffer);
12907 for (const auto &ParamAttr : ParamAttrs) {
12908 switch (ParamAttr.Kind) {
12909 case llvm::OpenMPIRBuilder::DeclareSimdKindTy::Linear:
12910 Out << 'l';
12911 break;
12912 case llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearRef:
12913 Out << 'R';
12914 break;
12915 case llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearUVal:
12916 Out << 'U';
12917 break;
12918 case llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearVal:
12919 Out << 'L';
12920 break;
12921 case llvm::OpenMPIRBuilder::DeclareSimdKindTy::Uniform:
12922 Out << 'u';
12923 break;
12924 case llvm::OpenMPIRBuilder::DeclareSimdKindTy::Vector:
12925 Out << 'v';
12926 break;
12927 }
12928 if (ParamAttr.HasVarStride)
12929 Out << "s" << ParamAttr.StrideOrArg;
12930 else if (ParamAttr.Kind ==
12931 llvm::OpenMPIRBuilder::DeclareSimdKindTy::Linear ||
12932 ParamAttr.Kind ==
12933 llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearRef ||
12934 ParamAttr.Kind ==
12935 llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearUVal ||
12936 ParamAttr.Kind ==
12937 llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearVal) {
12938 // Don't print the step value if it is not present or if it is
12939 // equal to 1.
12940 if (ParamAttr.StrideOrArg < 0)
12941 Out << 'n' << -ParamAttr.StrideOrArg;
12942 else if (ParamAttr.StrideOrArg != 1)
12943 Out << ParamAttr.StrideOrArg;
12944 }
12945
12946 if (!!ParamAttr.Alignment)
12947 Out << 'a' << ParamAttr.Alignment;
12948 }
12949
12950 return std::string(Out.str());
12951}
12952
12953void OpenMPIRBuilder::emitX86DeclareSimdFunction(
12954 llvm::Function *Fn, unsigned NumElts, const llvm::APSInt &VLENVal,
12955 llvm::ArrayRef<DeclareSimdAttrTy> ParamAttrs, DeclareSimdBranch Branch) {
12956 struct ISADataTy {
12957 char ISA;
12958 unsigned VecRegSize;
12959 };
12960 ISADataTy ISAData[] = {
12961 {.ISA: 'b', .VecRegSize: 128}, // SSE
12962 {.ISA: 'c', .VecRegSize: 256}, // AVX
12963 {.ISA: 'd', .VecRegSize: 256}, // AVX2
12964 {.ISA: 'e', .VecRegSize: 512}, // AVX512
12965 };
12966 llvm::SmallVector<char, 2> Masked;
12967 switch (Branch) {
12968 case DeclareSimdBranch::Undefined:
12969 Masked.push_back(Elt: 'N');
12970 Masked.push_back(Elt: 'M');
12971 break;
12972 case DeclareSimdBranch::Notinbranch:
12973 Masked.push_back(Elt: 'N');
12974 break;
12975 case DeclareSimdBranch::Inbranch:
12976 Masked.push_back(Elt: 'M');
12977 break;
12978 }
12979 for (char Mask : Masked) {
12980 for (const ISADataTy &Data : ISAData) {
12981 llvm::SmallString<256> Buffer;
12982 llvm::raw_svector_ostream Out(Buffer);
12983 Out << "_ZGV" << Data.ISA << Mask;
12984 if (!VLENVal) {
12985 assert(NumElts && "Non-zero simdlen/cdtsize expected");
12986 Out << llvm::APSInt::getUnsigned(X: Data.VecRegSize / NumElts);
12987 } else {
12988 Out << VLENVal;
12989 }
12990 Out << mangleVectorParameters(ParamAttrs);
12991 Out << '_' << Fn->getName();
12992 Fn->addFnAttr(Kind: Out.str());
12993 }
12994 }
12995}
12996
12997// Function used to add the attribute. The parameter `VLEN` is templated to
12998// allow the use of `x` when targeting scalable functions for SVE.
12999template <typename T>
13000static void addAArch64VectorName(T VLEN, StringRef LMask, StringRef Prefix,
13001 char ISA, StringRef ParSeq,
13002 StringRef MangledName, bool OutputBecomesInput,
13003 llvm::Function *Fn) {
13004 SmallString<256> Buffer;
13005 llvm::raw_svector_ostream Out(Buffer);
13006 Out << Prefix << ISA << LMask << VLEN;
13007 if (OutputBecomesInput)
13008 Out << 'v';
13009 Out << ParSeq << '_' << MangledName;
13010 Fn->addFnAttr(Kind: Out.str());
13011}
13012
13013// Helper function to generate the Advanced SIMD names depending on the value
13014// of the NDS when simdlen is not present.
13015static void addAArch64AdvSIMDNDSNames(unsigned NDS, StringRef Mask,
13016 StringRef Prefix, char ISA,
13017 StringRef ParSeq, StringRef MangledName,
13018 bool OutputBecomesInput,
13019 llvm::Function *Fn) {
13020 switch (NDS) {
13021 case 8:
13022 addAArch64VectorName(VLEN: 8, LMask: Mask, Prefix, ISA, ParSeq, MangledName,
13023 OutputBecomesInput, Fn);
13024 addAArch64VectorName(VLEN: 16, LMask: Mask, Prefix, ISA, ParSeq, MangledName,
13025 OutputBecomesInput, Fn);
13026 break;
13027 case 16:
13028 addAArch64VectorName(VLEN: 4, LMask: Mask, Prefix, ISA, ParSeq, MangledName,
13029 OutputBecomesInput, Fn);
13030 addAArch64VectorName(VLEN: 8, LMask: Mask, Prefix, ISA, ParSeq, MangledName,
13031 OutputBecomesInput, Fn);
13032 break;
13033 case 32:
13034 addAArch64VectorName(VLEN: 2, LMask: Mask, Prefix, ISA, ParSeq, MangledName,
13035 OutputBecomesInput, Fn);
13036 addAArch64VectorName(VLEN: 4, LMask: Mask, Prefix, ISA, ParSeq, MangledName,
13037 OutputBecomesInput, Fn);
13038 break;
13039 case 64:
13040 case 128:
13041 addAArch64VectorName(VLEN: 2, LMask: Mask, Prefix, ISA, ParSeq, MangledName,
13042 OutputBecomesInput, Fn);
13043 break;
13044 default:
13045 llvm_unreachable("Scalar type is too wide.");
13046 }
13047}
13048
13049/// Emit vector function attributes for AArch64, as defined in the AAVFABI.
13050void OpenMPIRBuilder::emitAArch64DeclareSimdFunction(
13051 llvm::Function *Fn, unsigned UserVLEN,
13052 llvm::ArrayRef<DeclareSimdAttrTy> ParamAttrs, DeclareSimdBranch Branch,
13053 char ISA, unsigned NarrowestDataSize, bool OutputBecomesInput) {
13054 assert((ISA == 'n' || ISA == 's') && "Expected ISA either 's' or 'n'.");
13055
13056 // Sort out parameter sequence.
13057 const std::string ParSeq = mangleVectorParameters(ParamAttrs);
13058 StringRef Prefix = "_ZGV";
13059 StringRef MangledName = Fn->getName();
13060
13061 // Generate simdlen from user input (if any).
13062 if (UserVLEN) {
13063 if (ISA == 's') {
13064 // SVE generates only a masked function.
13065 addAArch64VectorName(VLEN: UserVLEN, LMask: "M", Prefix, ISA, ParSeq, MangledName,
13066 OutputBecomesInput, Fn);
13067 return;
13068 }
13069
13070 switch (Branch) {
13071 case DeclareSimdBranch::Undefined:
13072 addAArch64VectorName(VLEN: UserVLEN, LMask: "N", Prefix, ISA, ParSeq, MangledName,
13073 OutputBecomesInput, Fn);
13074 addAArch64VectorName(VLEN: UserVLEN, LMask: "M", Prefix, ISA, ParSeq, MangledName,
13075 OutputBecomesInput, Fn);
13076 break;
13077 case DeclareSimdBranch::Inbranch:
13078 addAArch64VectorName(VLEN: UserVLEN, LMask: "M", Prefix, ISA, ParSeq, MangledName,
13079 OutputBecomesInput, Fn);
13080 break;
13081 case DeclareSimdBranch::Notinbranch:
13082 addAArch64VectorName(VLEN: UserVLEN, LMask: "N", Prefix, ISA, ParSeq, MangledName,
13083 OutputBecomesInput, Fn);
13084 break;
13085 }
13086 return;
13087 }
13088
13089 if (ISA == 's') {
13090 // SVE, section 3.4.1, item 1.
13091 addAArch64VectorName(VLEN: "x", LMask: "M", Prefix, ISA, ParSeq, MangledName,
13092 OutputBecomesInput, Fn);
13093 return;
13094 }
13095
13096 switch (Branch) {
13097 case DeclareSimdBranch::Undefined:
13098 addAArch64AdvSIMDNDSNames(NDS: NarrowestDataSize, Mask: "N", Prefix, ISA, ParSeq,
13099 MangledName, OutputBecomesInput, Fn);
13100 addAArch64AdvSIMDNDSNames(NDS: NarrowestDataSize, Mask: "M", Prefix, ISA, ParSeq,
13101 MangledName, OutputBecomesInput, Fn);
13102 break;
13103 case DeclareSimdBranch::Inbranch:
13104 addAArch64AdvSIMDNDSNames(NDS: NarrowestDataSize, Mask: "M", Prefix, ISA, ParSeq,
13105 MangledName, OutputBecomesInput, Fn);
13106 break;
13107 case DeclareSimdBranch::Notinbranch:
13108 addAArch64AdvSIMDNDSNames(NDS: NarrowestDataSize, Mask: "N", Prefix, ISA, ParSeq,
13109 MangledName, OutputBecomesInput, Fn);
13110 break;
13111 }
13112}
13113
13114//===----------------------------------------------------------------------===//
13115// OffloadEntriesInfoManager
13116//===----------------------------------------------------------------------===//
13117
13118bool OffloadEntriesInfoManager::empty() const {
13119 return OffloadEntriesTargetRegion.empty() &&
13120 OffloadEntriesDeviceGlobalVar.empty();
13121}
13122
13123unsigned OffloadEntriesInfoManager::getTargetRegionEntryInfoCount(
13124 const TargetRegionEntryInfo &EntryInfo) const {
13125 auto It = OffloadEntriesTargetRegionCount.find(
13126 x: getTargetRegionEntryCountKey(EntryInfo));
13127 if (It == OffloadEntriesTargetRegionCount.end())
13128 return 0;
13129 return It->second;
13130}
13131
13132void OffloadEntriesInfoManager::incrementTargetRegionEntryInfoCount(
13133 const TargetRegionEntryInfo &EntryInfo) {
13134 OffloadEntriesTargetRegionCount[getTargetRegionEntryCountKey(EntryInfo)] =
13135 EntryInfo.Count + 1;
13136}
13137
13138/// Initialize target region entry.
13139void OffloadEntriesInfoManager::initializeTargetRegionEntryInfo(
13140 const TargetRegionEntryInfo &EntryInfo, unsigned Order) {
13141 OffloadEntriesTargetRegion[EntryInfo] =
13142 OffloadEntryInfoTargetRegion(Order, /*Addr=*/nullptr, /*ID=*/nullptr,
13143 OMPTargetRegionEntryTargetRegion);
13144 ++OffloadingEntriesNum;
13145}
13146
13147void OffloadEntriesInfoManager::registerTargetRegionEntryInfo(
13148 TargetRegionEntryInfo EntryInfo, Constant *Addr, Constant *ID,
13149 OMPTargetRegionEntryKind Flags) {
13150 assert(EntryInfo.Count == 0 && "expected default EntryInfo");
13151
13152 // Update the EntryInfo with the next available count for this location.
13153 EntryInfo.Count = getTargetRegionEntryInfoCount(EntryInfo);
13154
13155 // If we are emitting code for a target, the entry is already initialized,
13156 // only has to be registered.
13157 if (OMPBuilder->Config.isTargetDevice()) {
13158 // This could happen if the device compilation is invoked standalone.
13159 if (!hasTargetRegionEntryInfo(EntryInfo)) {
13160 return;
13161 }
13162 auto &Entry = OffloadEntriesTargetRegion[EntryInfo];
13163 Entry.setAddress(Addr);
13164 Entry.setID(ID);
13165 Entry.setFlags(Flags);
13166 } else {
13167 if (Flags == OffloadEntriesInfoManager::OMPTargetRegionEntryTargetRegion &&
13168 hasTargetRegionEntryInfo(EntryInfo, /*IgnoreAddressId*/ true))
13169 return;
13170 assert(!hasTargetRegionEntryInfo(EntryInfo) &&
13171 "Target region entry already registered!");
13172 OffloadEntryInfoTargetRegion Entry(OffloadingEntriesNum, Addr, ID, Flags);
13173 OffloadEntriesTargetRegion[EntryInfo] = Entry;
13174 ++OffloadingEntriesNum;
13175 }
13176 incrementTargetRegionEntryInfoCount(EntryInfo);
13177}
13178
13179bool OffloadEntriesInfoManager::hasTargetRegionEntryInfo(
13180 TargetRegionEntryInfo EntryInfo, bool IgnoreAddressId) const {
13181
13182 // Update the EntryInfo with the next available count for this location.
13183 EntryInfo.Count = getTargetRegionEntryInfoCount(EntryInfo);
13184
13185 auto It = OffloadEntriesTargetRegion.find(x: EntryInfo);
13186 if (It == OffloadEntriesTargetRegion.end()) {
13187 return false;
13188 }
13189 // Fail if this entry is already registered.
13190 if (!IgnoreAddressId && (It->second.getAddress() || It->second.getID()))
13191 return false;
13192 return true;
13193}
13194
13195void OffloadEntriesInfoManager::actOnTargetRegionEntriesInfo(
13196 const OffloadTargetRegionEntryInfoActTy &Action) {
13197 // Scan all target region entries and perform the provided action.
13198 for (const auto &It : OffloadEntriesTargetRegion) {
13199 Action(It.first, It.second);
13200 }
13201}
13202
13203void OffloadEntriesInfoManager::initializeDeviceGlobalVarEntryInfo(
13204 StringRef Name, OMPTargetGlobalVarEntryKind Flags, unsigned Order) {
13205 OffloadEntriesDeviceGlobalVar.try_emplace(Key: Name, Args&: Order, Args&: Flags);
13206 ++OffloadingEntriesNum;
13207}
13208
13209void OffloadEntriesInfoManager::registerDeviceGlobalVarEntryInfo(
13210 StringRef VarName, Constant *Addr, int64_t VarSize,
13211 OMPTargetGlobalVarEntryKind Flags, GlobalValue::LinkageTypes Linkage) {
13212 if (OMPBuilder->Config.isTargetDevice()) {
13213 // This could happen if the device compilation is invoked standalone.
13214 if (!hasDeviceGlobalVarEntryInfo(VarName))
13215 return;
13216 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
13217 if (Entry.getAddress() && hasDeviceGlobalVarEntryInfo(VarName)) {
13218 if (Entry.getVarSize() == 0) {
13219 Entry.setVarSize(VarSize);
13220 Entry.setLinkage(Linkage);
13221 }
13222 return;
13223 }
13224 Entry.setVarSize(VarSize);
13225 Entry.setLinkage(Linkage);
13226 Entry.setAddress(Addr);
13227 } else {
13228 if (hasDeviceGlobalVarEntryInfo(VarName)) {
13229 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
13230 assert(Entry.isValid() && Entry.getFlags() == Flags &&
13231 "Entry not initialized!");
13232 if (Entry.getVarSize() == 0) {
13233 Entry.setVarSize(VarSize);
13234 Entry.setLinkage(Linkage);
13235 }
13236 return;
13237 }
13238 if (Flags == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect ||
13239 Flags ==
13240 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirectVTable)
13241 OffloadEntriesDeviceGlobalVar.try_emplace(Key: VarName, Args&: OffloadingEntriesNum,
13242 Args&: Addr, Args&: VarSize, Args&: Flags, Args&: Linkage,
13243 Args: VarName.str());
13244 else
13245 OffloadEntriesDeviceGlobalVar.try_emplace(
13246 Key: VarName, Args&: OffloadingEntriesNum, Args&: Addr, Args&: VarSize, Args&: Flags, Args&: Linkage, Args: "");
13247 ++OffloadingEntriesNum;
13248 }
13249}
13250
13251void OffloadEntriesInfoManager::actOnDeviceGlobalVarEntriesInfo(
13252 const OffloadDeviceGlobalVarEntryInfoActTy &Action) {
13253 // Scan all target region entries and perform the provided action.
13254 for (const auto &E : OffloadEntriesDeviceGlobalVar)
13255 Action(E.getKey(), E.getValue());
13256}
13257
13258//===----------------------------------------------------------------------===//
13259// CanonicalLoopInfo
13260//===----------------------------------------------------------------------===//
13261
13262void CanonicalLoopInfo::collectControlBlocks(
13263 SmallVectorImpl<BasicBlock *> &BBs) {
13264 // We only count those BBs as control block for which we do not need to
13265 // reverse the CFG, i.e. not the loop body which can contain arbitrary control
13266 // flow. For consistency, this also means we do not add the Body block, which
13267 // is just the entry to the body code.
13268 BBs.reserve(N: BBs.size() + 6);
13269 BBs.append(IL: {getPreheader(), Header, Cond, Latch, Exit, getAfter()});
13270}
13271
13272BasicBlock *CanonicalLoopInfo::getPreheader() const {
13273 assert(isValid() && "Requires a valid canonical loop");
13274 for (BasicBlock *Pred : predecessors(BB: Header)) {
13275 if (Pred != Latch)
13276 return Pred;
13277 }
13278 llvm_unreachable("Missing preheader");
13279}
13280
13281void CanonicalLoopInfo::setTripCount(Value *TripCount) {
13282 assert(isValid() && "Requires a valid canonical loop");
13283
13284 Instruction *CmpI = &getCond()->front();
13285 assert(isa<CmpInst>(CmpI) && "First inst must compare IV with TripCount");
13286 CmpI->setOperand(i: 1, Val: TripCount);
13287
13288#ifndef NDEBUG
13289 assertOK();
13290#endif
13291}
13292
13293void CanonicalLoopInfo::mapIndVar(
13294 llvm::function_ref<Value *(Instruction *)> Updater) {
13295 assert(isValid() && "Requires a valid canonical loop");
13296
13297 Instruction *OldIV = getIndVar();
13298
13299 // Record all uses excluding those introduced by the updater. Uses by the
13300 // CanonicalLoopInfo itself to keep track of the number of iterations are
13301 // excluded.
13302 SmallVector<Use *> ReplacableUses;
13303 for (Use &U : OldIV->uses()) {
13304 auto *User = dyn_cast<Instruction>(Val: U.getUser());
13305 if (!User)
13306 continue;
13307 if (User->getParent() == getCond())
13308 continue;
13309 if (User->getParent() == getLatch())
13310 continue;
13311 ReplacableUses.push_back(Elt: &U);
13312 }
13313
13314 // Run the updater that may introduce new uses
13315 Value *NewIV = Updater(OldIV);
13316
13317 // Replace the old uses with the value returned by the updater.
13318 for (Use *U : ReplacableUses)
13319 U->set(NewIV);
13320
13321#ifndef NDEBUG
13322 assertOK();
13323#endif
13324}
13325
13326void CanonicalLoopInfo::assertOK() const {
13327#ifndef NDEBUG
13328 // No constraints if this object currently does not describe a loop.
13329 if (!isValid())
13330 return;
13331
13332 BasicBlock *Preheader = getPreheader();
13333 BasicBlock *Body = getBody();
13334 BasicBlock *After = getAfter();
13335
13336 // Verify standard control-flow we use for OpenMP loops.
13337 assert(Preheader);
13338 assert(isa<UncondBrInst>(Preheader->getTerminator()) &&
13339 "Preheader must terminate with unconditional branch");
13340 assert(Preheader->getSingleSuccessor() == Header &&
13341 "Preheader must jump to header");
13342
13343 assert(Header);
13344 assert(isa<UncondBrInst>(Header->getTerminator()) &&
13345 "Header must terminate with unconditional branch");
13346 assert(Header->getSingleSuccessor() == Cond &&
13347 "Header must jump to exiting block");
13348
13349 assert(Cond);
13350 assert(Cond->getSinglePredecessor() == Header &&
13351 "Exiting block only reachable from header");
13352
13353 assert(isa<CondBrInst>(Cond->getTerminator()) &&
13354 "Exiting block must terminate with conditional branch");
13355 assert(cast<CondBrInst>(Cond->getTerminator())->getSuccessor(0) == Body &&
13356 "Exiting block's first successor jump to the body");
13357 assert(cast<CondBrInst>(Cond->getTerminator())->getSuccessor(1) == Exit &&
13358 "Exiting block's second successor must exit the loop");
13359
13360 assert(Body);
13361 assert(Body->getSinglePredecessor() == Cond &&
13362 "Body only reachable from exiting block");
13363 assert(!isa<PHINode>(Body->front()));
13364
13365 assert(Latch);
13366 assert(isa<UncondBrInst>(Latch->getTerminator()) &&
13367 "Latch must terminate with unconditional branch");
13368 assert(Latch->getSingleSuccessor() == Header && "Latch must jump to header");
13369 // TODO: To support simple redirecting of the end of the body code that has
13370 // multiple; introduce another auxiliary basic block like preheader and after.
13371 assert(Latch->getSinglePredecessor() != nullptr);
13372 assert(!isa<PHINode>(Latch->front()));
13373
13374 assert(Exit);
13375 assert(isa<UncondBrInst>(Exit->getTerminator()) &&
13376 "Exit block must terminate with unconditional branch");
13377 assert(Exit->getSingleSuccessor() == After &&
13378 "Exit block must jump to after block");
13379
13380 assert(After);
13381 assert(After->getSinglePredecessor() == Exit &&
13382 "After block only reachable from exit block");
13383 assert(After->empty() || !isa<PHINode>(After->front()));
13384
13385 Instruction *IndVar = getIndVar();
13386 assert(IndVar && "Canonical induction variable not found?");
13387 assert(isa<IntegerType>(IndVar->getType()) &&
13388 "Induction variable must be an integer");
13389 assert(cast<PHINode>(IndVar)->getParent() == Header &&
13390 "Induction variable must be a PHI in the loop header");
13391 assert(cast<PHINode>(IndVar)->getIncomingBlock(0) == Preheader);
13392 assert(
13393 cast<ConstantInt>(cast<PHINode>(IndVar)->getIncomingValue(0))->isZero());
13394 assert(cast<PHINode>(IndVar)->getIncomingBlock(1) == Latch);
13395
13396 auto *NextIndVar = cast<PHINode>(IndVar)->getIncomingValue(1);
13397 assert(cast<Instruction>(NextIndVar)->getParent() == Latch);
13398 assert(cast<BinaryOperator>(NextIndVar)->getOpcode() == BinaryOperator::Add);
13399 assert(cast<BinaryOperator>(NextIndVar)->getOperand(0) == IndVar);
13400 assert(cast<ConstantInt>(cast<BinaryOperator>(NextIndVar)->getOperand(1))
13401 ->isOne());
13402
13403 Value *TripCount = getTripCount();
13404 assert(TripCount && "Loop trip count not found?");
13405 assert(IndVar->getType() == TripCount->getType() &&
13406 "Trip count and induction variable must have the same type");
13407
13408 auto *CmpI = cast<CmpInst>(&Cond->front());
13409 assert(CmpI->getPredicate() == CmpInst::ICMP_ULT &&
13410 "Exit condition must be a signed less-than comparison");
13411 assert(CmpI->getOperand(0) == IndVar &&
13412 "Exit condition must compare the induction variable");
13413 assert(CmpI->getOperand(1) == TripCount &&
13414 "Exit condition must compare with the trip count");
13415#endif
13416}
13417
13418void CanonicalLoopInfo::invalidate() {
13419 Header = nullptr;
13420 Cond = nullptr;
13421 Latch = nullptr;
13422 Exit = nullptr;
13423}
13424