1//===------ NVPTXTagInvariantLoads.cpp - Tag invariant loads --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements invaraint load tagging. It traverses load instructions
10// in a function, and determines if each load can be tagged as invariant.
11//
12// We currently infer invariance for loads from
13// - constant global variables, and
14// - kernel function pointer params that are noalias (i.e. __restrict) and
15// never written to.
16//
17// TODO: Perform a more powerful invariance analysis (ideally IPO).
18//
19//===----------------------------------------------------------------------===//
20
21#include "NVPTXUtilities.h"
22#include "llvm/Analysis/ValueTracking.h"
23#include "llvm/IR/InstIterator.h"
24#include "llvm/IR/Instructions.h"
25#include "llvm/IR/Metadata.h"
26#include "llvm/Support/NVPTXAddrSpace.h"
27
28using namespace llvm;
29
30static bool isInvariantLoad(const Instruction *I, const Value *Ptr,
31 const bool IsKernelFn) {
32 // Don't bother with non-global loads
33 if (Ptr->getType()->getPointerAddressSpace() != NVPTXAS::ADDRESS_SPACE_GLOBAL)
34 return false;
35
36 // If the load is already marked as invariant, we don't need to do anything
37 if (I->getMetadata(KindID: LLVMContext::MD_invariant_load))
38 return false;
39
40 // We use getUnderlyingObjects() here instead of getUnderlyingObject()
41 // mainly because the former looks through phi nodes while the latter does
42 // not. We need to look through phi nodes to handle pointer induction
43 // variables.
44 SmallVector<const Value *, 8> Objs;
45 getUnderlyingObjects(V: Ptr, Objects&: Objs);
46
47 return all_of(Range&: Objs, P: [&](const Value *V) {
48 if (const auto *A = dyn_cast<const Argument>(Val: V))
49 return IsKernelFn && ((A->onlyReadsMemory() && A->hasNoAliasAttr()) ||
50 isParamGridConstant(*A));
51 if (const auto *GV = dyn_cast<const GlobalVariable>(Val: V))
52 return GV->isConstant();
53 return false;
54 });
55}
56
57static void markLoadsAsInvariant(Instruction *I) {
58 I->setMetadata(KindID: LLVMContext::MD_invariant_load,
59 Node: MDNode::get(Context&: I->getContext(), MDs: {}));
60}
61
62static bool tagInvariantLoads(Function &F) {
63 const bool IsKernelFn = isKernelFunction(F);
64
65 bool Changed = false;
66 for (auto &I : instructions(F)) {
67 if (auto *LI = dyn_cast<LoadInst>(Val: &I))
68 if (isInvariantLoad(I: LI, Ptr: LI->getPointerOperand(), IsKernelFn)) {
69 markLoadsAsInvariant(I: LI);
70 Changed = true;
71 }
72 if (auto *II = dyn_cast<IntrinsicInst>(Val: &I))
73 if (II->getIntrinsicID() == Intrinsic::masked_load &&
74 isInvariantLoad(I: II, Ptr: II->getOperand(i_nocapture: 0), IsKernelFn)) {
75 markLoadsAsInvariant(I: II);
76 Changed = true;
77 }
78 }
79 return Changed;
80}
81
82namespace {
83
84struct NVPTXTagInvariantLoadLegacyPass : public FunctionPass {
85 static char ID;
86
87 NVPTXTagInvariantLoadLegacyPass() : FunctionPass(ID) {}
88 bool runOnFunction(Function &F) override;
89};
90
91} // namespace
92
93INITIALIZE_PASS(NVPTXTagInvariantLoadLegacyPass, "nvptx-tag-invariant-loads",
94 "NVPTX Tag Invariant Loads", false, false)
95
96bool NVPTXTagInvariantLoadLegacyPass::runOnFunction(Function &F) {
97 return tagInvariantLoads(F);
98}
99
100char NVPTXTagInvariantLoadLegacyPass::ID = 0;
101
102FunctionPass *llvm::createNVPTXTagInvariantLoadsPass() {
103 return new NVPTXTagInvariantLoadLegacyPass();
104}
105
106PreservedAnalyses NVPTXTagInvariantLoadsPass::run(Function &F,
107 FunctionAnalysisManager &) {
108 return tagInvariantLoads(F) ? PreservedAnalyses::none()
109 : PreservedAnalyses::all();
110}
111