1//===- MemoryPromotion.cpp - Utilities for moving data across GPU memories ===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements utilities that allow one to create IR moving the data
10// across different levels of the GPU memory hierarchy.
11//
12//===----------------------------------------------------------------------===//
13
14#include "mlir/Dialect/GPU/Transforms/MemoryPromotion.h"
15
16#include "mlir/Dialect/Affine/LoopUtils.h"
17#include "mlir/Dialect/Arith/IR/Arith.h"
18#include "mlir/Dialect/GPU/IR/GPUDialect.h"
19#include "mlir/Dialect/MemRef/IR/MemRef.h"
20#include "mlir/Dialect/SCF/IR/SCF.h"
21
22using namespace mlir;
23using namespace mlir::gpu;
24
25/// Emits the (imperfect) loop nest performing the copy between "from" and "to"
26/// values using the bounds derived from the "from" value. Emits at least
27/// GPUDialect::getNumWorkgroupDimensions() loops, completing the nest with
28/// single-iteration loops. Maps the innermost loops to thread dimensions, in
29/// reverse order to enable access coalescing in the innermost loop.
30static void insertCopyLoops(ImplicitLocOpBuilder &b, Value from, Value to) {
31 auto memRefType = cast<MemRefType>(Val: from.getType());
32 auto rank = memRefType.getRank();
33
34 SmallVector<Value, 4> lbs, ubs, steps;
35 Value zero = arith::ConstantIndexOp::create(builder&: b, value: 0);
36 Value one = arith::ConstantIndexOp::create(builder&: b, value: 1);
37
38 // Make sure we have enough loops to use all thread dimensions, these trivial
39 // loops should be outermost and therefore inserted first.
40 if (rank < GPUDialect::getNumWorkgroupDimensions()) {
41 unsigned extraLoops = GPUDialect::getNumWorkgroupDimensions() - rank;
42 lbs.resize(N: extraLoops, NV: zero);
43 ubs.resize(N: extraLoops, NV: one);
44 steps.resize(N: extraLoops, NV: one);
45 }
46
47 // Add existing bounds.
48 lbs.append(NumInputs: rank, Elt: zero);
49 ubs.reserve(N: lbs.size());
50 steps.reserve(N: lbs.size());
51 for (auto idx = 0; idx < rank; ++idx) {
52 ubs.push_back(Elt: b.createOrFold<memref::DimOp>(args&: from, args&: idx));
53 steps.push_back(Elt: one);
54 }
55
56 // Obtain thread identifiers and block sizes, necessary to map to them.
57 auto indexType = b.getIndexType();
58 SmallVector<Value, 3> threadIds, blockDims;
59 for (auto dim : {gpu::Dimension::x, gpu::Dimension::y, gpu::Dimension::z}) {
60 threadIds.push_back(Elt: gpu::ThreadIdOp::create(builder&: b, resultType: indexType, dimension: dim));
61 blockDims.push_back(Elt: gpu::BlockDimOp::create(builder&: b, resultType: indexType, dimension: dim));
62 }
63
64 // Produce the loop nest with copies.
65 SmallVector<Value, 8> ivs(lbs.size());
66 mlir::scf::buildLoopNest(
67 builder&: b, loc: b.getLoc(), lbs, ubs, steps,
68 bodyBuilder: [&](OpBuilder &b, Location loc, ValueRange loopIvs) {
69 ivs.assign(in_start: loopIvs.begin(), in_end: loopIvs.end());
70 auto activeIvs = llvm::ArrayRef(ivs).take_back(N: rank);
71 Value loaded = memref::LoadOp::create(builder&: b, location: loc, memref: from, indices: activeIvs);
72 memref::StoreOp::create(builder&: b, location: loc, valueToStore: loaded, memref: to, indices: activeIvs);
73 });
74
75 // Map the innermost loops to threads in reverse order.
76 for (const auto &en :
77 llvm::enumerate(First: llvm::reverse(C: llvm::ArrayRef(ivs).take_back(
78 N: GPUDialect::getNumWorkgroupDimensions())))) {
79 Value v = en.value();
80 auto loop = cast<scf::ForOp>(Val: v.getParentRegion()->getParentOp());
81 affine::mapLoopToProcessorIds(forOp: loop, processorId: {threadIds[en.index()]},
82 numProcessors: {blockDims[en.index()]});
83 }
84}
85
86/// Emits the loop nests performing the copy to the designated location in the
87/// beginning of the region, and from the designated location immediately before
88/// the terminator of the first block of the region. The region is expected to
89/// have one block. This boils down to the following structure
90///
91/// ^bb(...):
92/// <loop-bound-computation>
93/// for %arg0 = ... to ... step ... {
94/// ...
95/// for %argN = <thread-id-x> to ... step <block-dim-x> {
96/// %0 = load %from[%arg0, ..., %argN]
97/// store %0, %to[%arg0, ..., %argN]
98/// }
99/// ...
100/// }
101/// gpu.barrier
102/// <... original body ...>
103/// gpu.barrier
104/// for %arg0 = ... to ... step ... {
105/// ...
106/// for %argN = <thread-id-x> to ... step <block-dim-x> {
107/// %1 = load %to[%arg0, ..., %argN]
108/// store %1, %from[%arg0, ..., %argN]
109/// }
110/// ...
111/// }
112///
113/// Inserts the barriers unconditionally since different threads may be copying
114/// values and reading them. An analysis would be required to eliminate barriers
115/// in case where value is only used by the thread that copies it. Both copies
116/// are inserted unconditionally, an analysis would be required to only copy
117/// live-in and live-out values when necessary. This copies the entire memref
118/// pointed to by "from". In case a smaller block would be sufficient, the
119/// caller can create a subview of the memref and promote it instead.
120static void insertCopies(Region &region, Location loc, Value from, Value to) {
121 auto fromType = cast<MemRefType>(Val: from.getType());
122 auto toType = cast<MemRefType>(Val: to.getType());
123 (void)fromType;
124 (void)toType;
125 assert(fromType.getShape() == toType.getShape());
126 assert(fromType.getRank() != 0);
127 assert(region.hasOneBlock() && "unstructured control flow not supported");
128
129 auto b = ImplicitLocOpBuilder::atBlockBegin(loc, block: &region.front());
130 insertCopyLoops(b, from, to);
131 gpu::BarrierOp::create(builder&: b);
132
133 b.setInsertionPoint(&region.front().back());
134 gpu::BarrierOp::create(builder&: b);
135 insertCopyLoops(b, from: to, to: from);
136}
137
138/// Promotes a function argument to workgroup memory in the given function. The
139/// copies will be inserted in the beginning and in the end of the function.
140void mlir::promoteToWorkgroupMemory(GPUFuncOp op, unsigned arg) {
141 Value value = op.getArgument(idx: arg);
142 auto type = dyn_cast<MemRefType>(Val: value.getType());
143 assert(type && type.hasStaticShape() && "can only promote memrefs");
144
145 // Get the type of the buffer in the workgroup memory.
146 auto workgroupMemoryAddressSpace = gpu::AddressSpaceAttr::get(
147 context: op->getContext(), value: gpu::AddressSpace::Workgroup);
148 auto bufferType = MemRefType::get(shape: type.getShape(), elementType: type.getElementType(),
149 layout: MemRefLayoutAttrInterface{},
150 memorySpace: Attribute(workgroupMemoryAddressSpace));
151 Value attribution = op.addWorkgroupAttribution(type: bufferType, loc: value.getLoc());
152
153 // Replace the uses first since only the original uses are currently present.
154 // Then insert the copies.
155 value.replaceAllUsesWith(newValue: attribution);
156 insertCopies(region&: op.getBody(), loc: op.getLoc(), from: value, to: attribution);
157}
158