1//===- NVPTXLowerAggrCopies.cpp - ------------------------------*- C++ -*--===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// \file
10// Lower aggregate copies, memset, memcpy, memmov intrinsics into loops when
11// the size is large or is not a compile-time constant.
12//
13//===----------------------------------------------------------------------===//
14
15#include "NVPTX.h"
16#include "llvm/Analysis/AliasAnalysis.h"
17#include "llvm/Analysis/TargetTransformInfo.h"
18#include "llvm/CodeGen/StackProtector.h"
19#include "llvm/IR/Constants.h"
20#include "llvm/IR/DataLayout.h"
21#include "llvm/IR/Function.h"
22#include "llvm/IR/IRBuilder.h"
23#include "llvm/IR/Instructions.h"
24#include "llvm/IR/IntrinsicInst.h"
25#include "llvm/IR/Intrinsics.h"
26#include "llvm/IR/LLVMContext.h"
27#include "llvm/IR/Module.h"
28#include "llvm/InitializePasses.h"
29#include "llvm/Transforms/Utils/BasicBlockUtils.h"
30#include "llvm/Transforms/Utils/LowerMemIntrinsics.h"
31
32#define DEBUG_TYPE "nvptx"
33
34using namespace llvm;
35
36static const unsigned MaxAggrCopySize = 128;
37
38static bool lowerAggrCopies(Function &F, const TargetTransformInfo &TTI,
39 AAResults &AA) {
40 SmallVector<LoadInst *, 4> AggrLoads;
41 SmallVector<MemIntrinsic *, 4> MemCalls;
42
43 const DataLayout &DL = F.getDataLayout();
44 LLVMContext &Context = F.getParent()->getContext();
45
46 // Collect all aggregate loads and mem* calls.
47 for (BasicBlock &BB : F) {
48 for (Instruction &I : BB) {
49 if (LoadInst *LI = dyn_cast<LoadInst>(Val: &I)) {
50 if (!LI->hasOneUse())
51 continue;
52
53 if (DL.getTypeStoreSize(Ty: LI->getType()) < MaxAggrCopySize)
54 continue;
55
56 if (StoreInst *SI = dyn_cast<StoreInst>(Val: LI->user_back())) {
57 if (SI->getOperand(i_nocapture: 0) != LI)
58 continue;
59 AggrLoads.push_back(Elt: LI);
60 }
61 } else if (MemIntrinsic *IntrCall = dyn_cast<MemIntrinsic>(Val: &I)) {
62 // Convert intrinsic calls with variable size or with constant size
63 // larger than the MaxAggrCopySize threshold.
64 if (ConstantInt *LenCI = dyn_cast<ConstantInt>(Val: IntrCall->getLength())) {
65 if (LenCI->getZExtValue() >= MaxAggrCopySize) {
66 MemCalls.push_back(Elt: IntrCall);
67 }
68 } else {
69 MemCalls.push_back(Elt: IntrCall);
70 }
71 }
72 }
73 }
74
75 if (AggrLoads.size() == 0 && MemCalls.size() == 0) {
76 return false;
77 }
78
79 //
80 // Do the transformation of an aggr load/copy/set to a loop
81 //
82 for (LoadInst *LI : AggrLoads) {
83 auto *SI = cast<StoreInst>(Val: *LI->user_begin());
84 Value *SrcAddr = LI->getOperand(i_nocapture: 0);
85 Value *DstAddr = SI->getOperand(i_nocapture: 1);
86 unsigned NumLoads = DL.getTypeStoreSize(Ty: LI->getType());
87 ConstantInt *CopyLen =
88 ConstantInt::get(Ty: Type::getInt32Ty(C&: Context), V: NumLoads);
89
90 LocationSize Size = LocationSize::precise(Value: NumLoads);
91 if (AA.isNoAlias(LocA: MemoryLocation(SrcAddr, Size),
92 LocB: MemoryLocation(DstAddr, Size))) {
93 // No overlap: emit a plain memcpy loop. Expand the loop here (rather
94 // than emitting a memcpy intrinsic and letting the code below expand it)
95 // so we can pass CanOverlap = false; expandMemCpyAsLoop would
96 // conservatively assume overlap.
97 createMemCpyLoopKnownSize(/* ConvertedInst */ InsertBefore: SI,
98 /* SrcAddr */ SrcAddr, /* DstAddr */ DstAddr,
99 /* CopyLen */ CopyLen,
100 /* SrcAlign */ LI->getAlign(),
101 /* DestAlign */ SI->getAlign(),
102 /* SrcIsVolatile */ LI->isVolatile(),
103 /* DstIsVolatile */ SI->isVolatile(),
104 /* CanOverlap */ false, TTI);
105 } else {
106 // May alias: lower as a memmove, which picks the copy direction at
107 // runtime. Emit the intrinsic here and let the loop below expand it.
108 //
109 // The pointers may alias even if they're in different address spaces
110 // (e.g. the generic addrspace may alias global). If they're in
111 // different addrspaces, cast to the generic space first, because
112 // expandMemMoveAsLoop needs to compare the pointer values to determine
113 // the copy direction.
114 IRBuilder<> Builder(SI);
115 unsigned SrcAS = LI->getPointerAddressSpace();
116 unsigned DstAS = SI->getPointerAddressSpace();
117 if (SrcAS != DstAS) {
118 PointerType *GenericPtrTy =
119 PointerType::get(C&: Context, AddressSpace: NVPTXAS::ADDRESS_SPACE_GENERIC);
120 SrcAddr = Builder.CreateAddrSpaceCast(V: SrcAddr, DestTy: GenericPtrTy);
121 DstAddr = Builder.CreateAddrSpaceCast(V: DstAddr, DestTy: GenericPtrTy);
122 }
123 MemCalls.push_back(Elt: cast<MemMoveInst>(Val: Builder.CreateMemMove(
124 Dst: DstAddr, DstAlign: SI->getAlign(), Src: SrcAddr, SrcAlign: LI->getAlign(), Size: CopyLen,
125 isVolatile: LI->isVolatile() || SI->isVolatile())));
126 }
127
128 SI->eraseFromParent();
129 LI->eraseFromParent();
130 }
131
132 // Transform mem* intrinsic calls.
133 for (MemIntrinsic *MemCall : MemCalls) {
134 bool Expanded = true;
135 if (MemCpyInst *Memcpy = dyn_cast<MemCpyInst>(Val: MemCall)) {
136 expandMemCpyAsLoop(MemCpy: Memcpy, TTI);
137 } else if (MemMoveInst *Memmove = dyn_cast<MemMoveInst>(Val: MemCall)) {
138 Expanded = expandMemMoveAsLoop(MemMove: Memmove, TTI);
139 } else if (MemSetInst *Memset = dyn_cast<MemSetInst>(Val: MemCall)) {
140 expandMemSetAsLoop(MemSet: Memset, TTI);
141 }
142 if (Expanded)
143 MemCall->eraseFromParent();
144 }
145
146 return true;
147}
148
149namespace {
150
151struct NVPTXLowerAggrCopiesLegacyPass : public FunctionPass {
152 static char ID;
153
154 NVPTXLowerAggrCopiesLegacyPass() : FunctionPass(ID) {}
155
156 void getAnalysisUsage(AnalysisUsage &AU) const override {
157 AU.addPreserved<StackProtector>();
158 AU.addRequired<TargetTransformInfoWrapperPass>();
159 AU.addRequired<AAResultsWrapperPass>();
160 }
161
162 bool runOnFunction(Function &F) override {
163 return lowerAggrCopies(
164 F, TTI: getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F),
165 AA&: getAnalysis<AAResultsWrapperPass>().getAAResults());
166 }
167
168 StringRef getPassName() const override {
169 return "Lower aggregate copies/intrinsics into loops";
170 }
171};
172
173char NVPTXLowerAggrCopiesLegacyPass::ID = 0;
174
175} // namespace
176
177INITIALIZE_PASS_BEGIN(
178 NVPTXLowerAggrCopiesLegacyPass, "nvptx-lower-aggr-copies",
179 "Lower aggregate copies, and llvm.mem* intrinsics into loops", false, false)
180INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
181INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
182INITIALIZE_PASS_END(
183 NVPTXLowerAggrCopiesLegacyPass, "nvptx-lower-aggr-copies",
184 "Lower aggregate copies, and llvm.mem* intrinsics into loops", false, false)
185
186FunctionPass *llvm::createNVPTXLowerAggrCopiesLegacyPass() {
187 return new NVPTXLowerAggrCopiesLegacyPass();
188}
189
190PreservedAnalyses NVPTXLowerAggrCopiesPass::run(Function &F,
191 FunctionAnalysisManager &FAM) {
192 if (!lowerAggrCopies(F, TTI: FAM.getResult<TargetIRAnalysis>(IR&: F),
193 AA&: FAM.getResult<AAManager>(IR&: F)))
194 return PreservedAnalyses::all();
195 // Copies are expanded into loops, so the CFG is not preserved.
196 PreservedAnalyses PA;
197 PA.preserve<SSPLayoutAnalysis>();
198 return PA;
199}
200