1//===- NVPTXLowerAggrCopies.cpp - ------------------------------*- C++ -*--===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// \file
10// Lower aggregate copies, memset, memcpy, memmov intrinsics into loops when
11// the size is large or is not a compile-time constant.
12//
13//===----------------------------------------------------------------------===//
14
15#include "NVPTXLowerAggrCopies.h"
16#include "NVPTX.h"
17#include "llvm/Analysis/AliasAnalysis.h"
18#include "llvm/Analysis/TargetTransformInfo.h"
19#include "llvm/CodeGen/StackProtector.h"
20#include "llvm/IR/Constants.h"
21#include "llvm/IR/DataLayout.h"
22#include "llvm/IR/Function.h"
23#include "llvm/IR/IRBuilder.h"
24#include "llvm/IR/Instructions.h"
25#include "llvm/IR/IntrinsicInst.h"
26#include "llvm/IR/Intrinsics.h"
27#include "llvm/IR/LLVMContext.h"
28#include "llvm/IR/Module.h"
29#include "llvm/InitializePasses.h"
30#include "llvm/Transforms/Utils/BasicBlockUtils.h"
31#include "llvm/Transforms/Utils/LowerMemIntrinsics.h"
32
33#define DEBUG_TYPE "nvptx"
34
35using namespace llvm;
36
37namespace {
38
39// actual analysis class, which is a functionpass
40struct NVPTXLowerAggrCopies : public FunctionPass {
41 static char ID;
42
43 NVPTXLowerAggrCopies() : FunctionPass(ID) {}
44
45 void getAnalysisUsage(AnalysisUsage &AU) const override {
46 AU.addPreserved<StackProtector>();
47 AU.addRequired<TargetTransformInfoWrapperPass>();
48 AU.addRequired<AAResultsWrapperPass>();
49 }
50
51 bool runOnFunction(Function &F) override;
52
53 static const unsigned MaxAggrCopySize = 128;
54
55 StringRef getPassName() const override {
56 return "Lower aggregate copies/intrinsics into loops";
57 }
58};
59
60char NVPTXLowerAggrCopies::ID = 0;
61
62bool NVPTXLowerAggrCopies::runOnFunction(Function &F) {
63 SmallVector<LoadInst *, 4> AggrLoads;
64 SmallVector<MemIntrinsic *, 4> MemCalls;
65
66 const DataLayout &DL = F.getDataLayout();
67 LLVMContext &Context = F.getParent()->getContext();
68 const TargetTransformInfo &TTI =
69 getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
70 AAResults &AA = getAnalysis<AAResultsWrapperPass>().getAAResults();
71
72 // Collect all aggregate loads and mem* calls.
73 for (BasicBlock &BB : F) {
74 for (Instruction &I : BB) {
75 if (LoadInst *LI = dyn_cast<LoadInst>(Val: &I)) {
76 if (!LI->hasOneUse())
77 continue;
78
79 if (DL.getTypeStoreSize(Ty: LI->getType()) < MaxAggrCopySize)
80 continue;
81
82 if (StoreInst *SI = dyn_cast<StoreInst>(Val: LI->user_back())) {
83 if (SI->getOperand(i_nocapture: 0) != LI)
84 continue;
85 AggrLoads.push_back(Elt: LI);
86 }
87 } else if (MemIntrinsic *IntrCall = dyn_cast<MemIntrinsic>(Val: &I)) {
88 // Convert intrinsic calls with variable size or with constant size
89 // larger than the MaxAggrCopySize threshold.
90 if (ConstantInt *LenCI = dyn_cast<ConstantInt>(Val: IntrCall->getLength())) {
91 if (LenCI->getZExtValue() >= MaxAggrCopySize) {
92 MemCalls.push_back(Elt: IntrCall);
93 }
94 } else {
95 MemCalls.push_back(Elt: IntrCall);
96 }
97 }
98 }
99 }
100
101 if (AggrLoads.size() == 0 && MemCalls.size() == 0) {
102 return false;
103 }
104
105 //
106 // Do the transformation of an aggr load/copy/set to a loop
107 //
108 for (LoadInst *LI : AggrLoads) {
109 auto *SI = cast<StoreInst>(Val: *LI->user_begin());
110 Value *SrcAddr = LI->getOperand(i_nocapture: 0);
111 Value *DstAddr = SI->getOperand(i_nocapture: 1);
112 unsigned NumLoads = DL.getTypeStoreSize(Ty: LI->getType());
113 ConstantInt *CopyLen =
114 ConstantInt::get(Ty: Type::getInt32Ty(C&: Context), V: NumLoads);
115
116 LocationSize Size = LocationSize::precise(Value: NumLoads);
117 if (AA.isNoAlias(LocA: MemoryLocation(SrcAddr, Size),
118 LocB: MemoryLocation(DstAddr, Size))) {
119 // No overlap: emit a plain memcpy loop. Expand the loop here (rather
120 // than emitting a memcpy intrinsic and letting the code below expand it)
121 // so we can pass CanOverlap = false; expandMemCpyAsLoop would
122 // conservatively assume overlap.
123 createMemCpyLoopKnownSize(/* ConvertedInst */ InsertBefore: SI,
124 /* SrcAddr */ SrcAddr, /* DstAddr */ DstAddr,
125 /* CopyLen */ CopyLen,
126 /* SrcAlign */ LI->getAlign(),
127 /* DestAlign */ SI->getAlign(),
128 /* SrcIsVolatile */ LI->isVolatile(),
129 /* DstIsVolatile */ SI->isVolatile(),
130 /* CanOverlap */ false, TTI);
131 } else {
132 // May alias: lower as a memmove, which picks the copy direction at
133 // runtime. Emit the intrinsic here and let the loop below expand it.
134 //
135 // The pointers may alias even if they're in different address spaces
136 // (e.g. the generic addrspace may alias global). If they're in
137 // different addrspaces, cast to the generic space first, because
138 // expandMemMoveAsLoop needs to compare the pointer values to determine
139 // the copy direction.
140 IRBuilder<> Builder(SI);
141 unsigned SrcAS = LI->getPointerAddressSpace();
142 unsigned DstAS = SI->getPointerAddressSpace();
143 if (SrcAS != DstAS) {
144 PointerType *GenericPtrTy =
145 PointerType::get(C&: Context, AddressSpace: NVPTXAS::ADDRESS_SPACE_GENERIC);
146 SrcAddr = Builder.CreateAddrSpaceCast(V: SrcAddr, DestTy: GenericPtrTy);
147 DstAddr = Builder.CreateAddrSpaceCast(V: DstAddr, DestTy: GenericPtrTy);
148 }
149 MemCalls.push_back(Elt: cast<MemMoveInst>(Val: Builder.CreateMemMove(
150 Dst: DstAddr, DstAlign: SI->getAlign(), Src: SrcAddr, SrcAlign: LI->getAlign(), Size: CopyLen,
151 isVolatile: LI->isVolatile() || SI->isVolatile())));
152 }
153
154 SI->eraseFromParent();
155 LI->eraseFromParent();
156 }
157
158 // Transform mem* intrinsic calls.
159 for (MemIntrinsic *MemCall : MemCalls) {
160 bool Expanded = true;
161 if (MemCpyInst *Memcpy = dyn_cast<MemCpyInst>(Val: MemCall)) {
162 expandMemCpyAsLoop(MemCpy: Memcpy, TTI);
163 } else if (MemMoveInst *Memmove = dyn_cast<MemMoveInst>(Val: MemCall)) {
164 Expanded = expandMemMoveAsLoop(MemMove: Memmove, TTI);
165 } else if (MemSetInst *Memset = dyn_cast<MemSetInst>(Val: MemCall)) {
166 expandMemSetAsLoop(MemSet: Memset, TTI);
167 }
168 if (Expanded)
169 MemCall->eraseFromParent();
170 }
171
172 return true;
173}
174
175} // namespace
176
177INITIALIZE_PASS_BEGIN(
178 NVPTXLowerAggrCopies, "nvptx-lower-aggr-copies",
179 "Lower aggregate copies, and llvm.mem* intrinsics into loops", false, false)
180INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
181INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
182INITIALIZE_PASS_END(
183 NVPTXLowerAggrCopies, "nvptx-lower-aggr-copies",
184 "Lower aggregate copies, and llvm.mem* intrinsics into loops", false, false)
185
186FunctionPass *llvm::createLowerAggrCopies() {
187 return new NVPTXLowerAggrCopies();
188}
189