1//===------ NVPTXTagInvariantLoads.cpp - Tag invariant loads --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements invaraint load tagging. It traverses load instructions
10// in a function, and determines if each load can be tagged as invariant.
11//
12// We currently infer invariance for loads from
13// - constant global variables, and
14// - kernel function pointer params that are noalias (i.e. __restrict) and
15// never written to.
16//
17// TODO: Perform a more powerful invariance analysis (ideally IPO).
18//
19//===----------------------------------------------------------------------===//
20
21#include "NVPTX.h"
22#include "NVVMProperties.h"
23#include "llvm/Analysis/ValueTracking.h"
24#include "llvm/IR/GlobalVariable.h"
25#include "llvm/IR/InstIterator.h"
26#include "llvm/IR/Instructions.h"
27#include "llvm/IR/IntrinsicInst.h"
28#include "llvm/IR/Metadata.h"
29#include "llvm/Support/NVPTXAddrSpace.h"
30
31using namespace llvm;
32
33static bool isInvariantLoad(const Instruction *I, const Value *Ptr,
34 const bool IsKernelFn) {
35 // Don't bother with non-global loads
36 if (Ptr->getType()->getPointerAddressSpace() != NVPTXAS::ADDRESS_SPACE_GLOBAL)
37 return false;
38
39 // If the load is already marked as invariant, we don't need to do anything
40 if (I->getMetadata(KindID: LLVMContext::MD_invariant_load))
41 return false;
42
43 // We use getUnderlyingObjects() here instead of getUnderlyingObject()
44 // mainly because the former looks through phi nodes while the latter does
45 // not. We need to look through phi nodes to handle pointer induction
46 // variables.
47 SmallVector<const Value *, 8> Objs;
48 getUnderlyingObjects(V: Ptr, Objects&: Objs);
49
50 return all_of(Range&: Objs, P: [&](const Value *V) {
51 if (const auto *A = dyn_cast<const Argument>(Val: V))
52 return IsKernelFn && ((A->onlyReadsMemory() && A->hasNoAliasAttr()) ||
53 isParamGridConstant(*A));
54 if (const auto *GV = dyn_cast<const GlobalVariable>(Val: V))
55 return GV->isConstant();
56 return false;
57 });
58}
59
60static void markLoadsAsInvariant(Instruction *I) {
61 I->setMetadata(KindID: LLVMContext::MD_invariant_load,
62 Node: MDNode::get(Context&: I->getContext(), MDs: {}));
63}
64
65static bool tagInvariantLoads(Function &F) {
66 const bool IsKernelFn = isKernelFunction(F);
67
68 bool Changed = false;
69 for (auto &I : instructions(F)) {
70 if (auto *LI = dyn_cast<LoadInst>(Val: &I))
71 if (isInvariantLoad(I: LI, Ptr: LI->getPointerOperand(), IsKernelFn)) {
72 markLoadsAsInvariant(I: LI);
73 Changed = true;
74 }
75 if (auto *II = dyn_cast<IntrinsicInst>(Val: &I))
76 if (II->getIntrinsicID() == Intrinsic::masked_load &&
77 isInvariantLoad(I: II, Ptr: II->getOperand(i_nocapture: 0), IsKernelFn)) {
78 markLoadsAsInvariant(I: II);
79 Changed = true;
80 }
81 }
82 return Changed;
83}
84
85namespace {
86
87struct NVPTXTagInvariantLoadLegacyPass : public FunctionPass {
88 static char ID;
89
90 NVPTXTagInvariantLoadLegacyPass() : FunctionPass(ID) {}
91 bool runOnFunction(Function &F) override;
92};
93
94} // namespace
95
96INITIALIZE_PASS(NVPTXTagInvariantLoadLegacyPass, "nvptx-tag-invariant-loads",
97 "NVPTX Tag Invariant Loads", false, false)
98
99bool NVPTXTagInvariantLoadLegacyPass::runOnFunction(Function &F) {
100 return tagInvariantLoads(F);
101}
102
103char NVPTXTagInvariantLoadLegacyPass::ID = 0;
104
105FunctionPass *llvm::createNVPTXTagInvariantLoadsPass() {
106 return new NVPTXTagInvariantLoadLegacyPass();
107}
108
109PreservedAnalyses NVPTXTagInvariantLoadsPass::run(Function &F,
110 FunctionAnalysisManager &) {
111 return tagInvariantLoads(F) ? PreservedAnalyses::none()
112 : PreservedAnalyses::all();
113}
114