1//===- UnifyAliasedResourcePass.cpp - Pass to Unify Aliased Resources -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements a pass that unifies access of multiple aliased resources
10// into access of one single resource.
11//
12//===----------------------------------------------------------------------===//
13
14#include "mlir/Dialect/SPIRV/Transforms/Passes.h"
15
16#include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
17#include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
18#include "mlir/Dialect/SPIRV/IR/SPIRVTypes.h"
19#include "mlir/Dialect/SPIRV/IR/TargetAndABI.h"
20#include "mlir/IR/Builders.h"
21#include "mlir/IR/BuiltinAttributes.h"
22#include "mlir/IR/BuiltinTypes.h"
23#include "mlir/IR/SymbolTable.h"
24#include "mlir/Transforms/DialectConversion.h"
25#include "llvm/ADT/DenseMap.h"
26#include "llvm/ADT/STLExtras.h"
27#include "llvm/Support/Alignment.h"
28#include <iterator>
29
30namespace mlir {
31namespace spirv {
32#define GEN_PASS_DEF_SPIRVUNIFYALIASEDRESOURCEPASS
33#include "mlir/Dialect/SPIRV/Transforms/Passes.h.inc"
34} // namespace spirv
35} // namespace mlir
36
37using namespace mlir;
38
39//===----------------------------------------------------------------------===//
40// Utility functions
41//===----------------------------------------------------------------------===//
42
43using Descriptor = std::pair<uint32_t, uint32_t>; // (set #, binding #)
44using AliasedResourceMap =
45 DenseMap<Descriptor, SmallVector<spirv::GlobalVariableOp>>;
46
47/// Collects all aliased resources in the given SPIR-V `moduleOp`.
48static AliasedResourceMap collectAliasedResources(spirv::ModuleOp moduleOp) {
49 AliasedResourceMap aliasedResources;
50 moduleOp->walk(callback: [&aliasedResources](spirv::GlobalVariableOp varOp) {
51 if (varOp->getDiscardableAttrOfType<UnitAttr>(name: "aliased")) {
52 std::optional<uint32_t> set = varOp.getDescriptorSet();
53 std::optional<uint32_t> binding = varOp.getBinding();
54 if (set && binding)
55 aliasedResources[{*set, *binding}].push_back(Elt: varOp);
56 }
57 });
58 return aliasedResources;
59}
60
61/// Returns the element type if the given `type` is a runtime array resource:
62/// `!spirv.ptr<!spirv.struct<!spirv.rtarray<...>>>`. Returns null type
63/// otherwise.
64static Type getRuntimeArrayElementType(Type type) {
65 auto ptrType = dyn_cast<spirv::PointerType>(Val&: type);
66 if (!ptrType)
67 return {};
68
69 auto structType = dyn_cast<spirv::StructType>(Val: ptrType.getPointeeType());
70 if (!structType || structType.getNumElements() != 1)
71 return {};
72
73 auto rtArrayType =
74 dyn_cast<spirv::RuntimeArrayType>(Val: structType.getElementType(0));
75 if (!rtArrayType)
76 return {};
77
78 return rtArrayType.getElementType();
79}
80
81/// Given a list of resource element `types`, returns the index of the canonical
82/// resource that all resources should be unified into. Returns std::nullopt if
83/// unable to unify.
84static std::optional<int>
85deduceCanonicalResource(ArrayRef<spirv::SPIRVType> types) {
86 // scalarNumBits: contains all resources' scalar types' bit counts.
87 // vectorNumBits: only contains resources whose element types are vectors.
88 // vectorIndices: each vector's original index in `types`.
89 SmallVector<int> scalarNumBits, vectorNumBits, vectorIndices;
90 scalarNumBits.reserve(N: types.size());
91 vectorNumBits.reserve(N: types.size());
92 vectorIndices.reserve(N: types.size());
93
94 for (const auto &indexedTypes : llvm::enumerate(First&: types)) {
95 spirv::SPIRVType type = indexedTypes.value();
96 assert(type.isScalarOrVector());
97 if (auto vectorType = dyn_cast<VectorType>(Val&: type)) {
98 if (vectorType.getNumElements() % 2 != 0)
99 return std::nullopt; // Odd-sized vector has special layout
100 // requirements.
101
102 std::optional<int64_t> numBytes = type.getSizeInBytes();
103 if (!numBytes)
104 return std::nullopt;
105
106 scalarNumBits.push_back(
107 Elt: vectorType.getElementType().getIntOrFloatBitWidth());
108 vectorNumBits.push_back(Elt: *numBytes * 8);
109 vectorIndices.push_back(Elt: indexedTypes.index());
110 } else {
111 scalarNumBits.push_back(Elt: type.getIntOrFloatBitWidth());
112 }
113 }
114
115 if (!vectorNumBits.empty()) {
116 // Choose the *vector* with the smallest bitwidth as the canonical resource,
117 // so that we can still keep vectorized load/store and avoid partial updates
118 // to large vectors.
119 auto *minVal = llvm::min_element(Range&: vectorNumBits);
120 // Make sure that the canonical resource's bitwidth is divisible by others.
121 // With out this, we cannot properly adjust the index later.
122 if (llvm::any_of(Range&: vectorNumBits,
123 P: [&](int bits) { return bits % *minVal != 0; }))
124 return std::nullopt;
125
126 // Require all scalar type bit counts to be a multiple of the chosen
127 // vector's primitive type to avoid reading/writing subcomponents.
128 int index = vectorIndices[std::distance(first: vectorNumBits.begin(), last: minVal)];
129 int baseNumBits = scalarNumBits[index];
130 if (llvm::any_of(Range&: scalarNumBits,
131 P: [&](int bits) { return bits % baseNumBits != 0; }))
132 return std::nullopt;
133
134 return index;
135 }
136
137 // All element types are scalars. Then choose the smallest bitwidth as the
138 // cannonical resource to avoid subcomponent load/store.
139 auto *minVal = llvm::min_element(Range&: scalarNumBits);
140 if (llvm::any_of(Range&: scalarNumBits,
141 P: [minVal](int64_t bit) { return bit % *minVal != 0; }))
142 return std::nullopt;
143 return std::distance(first: scalarNumBits.begin(), last: minVal);
144}
145
146static bool areSameBitwidthScalarOrVectorType(Type a, Type b) {
147 if (auto va = dyn_cast<VectorType>(Val&: a)) {
148 auto vb = dyn_cast<VectorType>(Val&: b);
149 return vb && va.getNumElements() * va.getElementTypeBitWidth() ==
150 vb.getNumElements() * vb.getElementTypeBitWidth();
151 }
152 return a.isIntOrFloat() && b.isIntOrFloat() &&
153 a.getIntOrFloatBitWidth() == b.getIntOrFloatBitWidth();
154}
155
156//===----------------------------------------------------------------------===//
157// Analysis
158//===----------------------------------------------------------------------===//
159
160namespace {
161/// A class for analyzing aliased resources.
162///
163/// Resources are expected to be spirv.GlobalVarible that has a descriptor set
164/// and binding number. Such resources are of the type
165/// `!spirv.ptr<!spirv.struct<...>>` per Vulkan requirements.
166///
167/// Right now, we only support the case that there is a single runtime array
168/// inside the struct.
169class ResourceAliasAnalysis {
170public:
171 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(ResourceAliasAnalysis)
172
173 explicit ResourceAliasAnalysis(Operation *);
174
175 /// Returns true if the given `op` can be rewritten to use a canonical
176 /// resource.
177 bool shouldUnify(Operation *op) const;
178
179 /// Returns all descriptors and their corresponding aliased resources.
180 const AliasedResourceMap &getResourceMap() const { return resourceMap; }
181
182 /// Returns the canonical resource for the given descriptor/variable.
183 spirv::GlobalVariableOp
184 getCanonicalResource(const Descriptor &descriptor) const;
185 spirv::GlobalVariableOp
186 getCanonicalResource(spirv::GlobalVariableOp varOp) const;
187
188 /// Returns the element type for the given variable.
189 spirv::SPIRVType getElementType(spirv::GlobalVariableOp varOp) const;
190
191private:
192 /// Given the descriptor and aliased resources bound to it, analyze whether we
193 /// can unify them and record if so.
194 void recordIfUnifiable(const Descriptor &descriptor,
195 ArrayRef<spirv::GlobalVariableOp> resources);
196
197 /// Mapping from a descriptor to all aliased resources bound to it.
198 AliasedResourceMap resourceMap;
199
200 /// Mapping from a descriptor to the chosen canonical resource.
201 DenseMap<Descriptor, spirv::GlobalVariableOp> canonicalResourceMap;
202
203 /// Mapping from an aliased resource to its descriptor.
204 DenseMap<spirv::GlobalVariableOp, Descriptor> descriptorMap;
205
206 /// Mapping from an aliased resource to its element (scalar/vector) type.
207 DenseMap<spirv::GlobalVariableOp, spirv::SPIRVType> elementTypeMap;
208};
209} // namespace
210
211ResourceAliasAnalysis::ResourceAliasAnalysis(Operation *root) {
212 // Collect all aliased resources first and put them into different sets
213 // according to the descriptor.
214 AliasedResourceMap aliasedResources =
215 collectAliasedResources(moduleOp: cast<spirv::ModuleOp>(Val: root));
216
217 // For each resource set, analyze whether we can unify; if so, try to identify
218 // a canonical resource, whose element type has the largest bitwidth.
219 for (const auto &descriptorResource : aliasedResources) {
220 recordIfUnifiable(descriptor: descriptorResource.first, resources: descriptorResource.second);
221 }
222}
223
224bool ResourceAliasAnalysis::shouldUnify(Operation *op) const {
225 if (!op)
226 return false;
227
228 if (auto varOp = dyn_cast<spirv::GlobalVariableOp>(Val: op)) {
229 auto canonicalOp = getCanonicalResource(varOp);
230 return canonicalOp && varOp != canonicalOp;
231 }
232 if (auto addressOp = dyn_cast<spirv::AddressOfOp>(Val: op)) {
233 auto moduleOp = addressOp->getParentOfType<spirv::ModuleOp>();
234 auto *varOp =
235 SymbolTable::lookupSymbolIn(op: moduleOp, symbol: addressOp.getVariable());
236 return shouldUnify(op: varOp);
237 }
238
239 if (auto acOp = dyn_cast<spirv::AccessChainOp>(Val: op))
240 return shouldUnify(op: acOp.getBasePtr().getDefiningOp());
241 if (auto loadOp = dyn_cast<spirv::LoadOp>(Val: op))
242 return shouldUnify(op: loadOp.getPtr().getDefiningOp());
243 if (auto storeOp = dyn_cast<spirv::StoreOp>(Val: op))
244 return shouldUnify(op: storeOp.getPtr().getDefiningOp());
245
246 return false;
247}
248
249spirv::GlobalVariableOp ResourceAliasAnalysis::getCanonicalResource(
250 const Descriptor &descriptor) const {
251 auto varIt = canonicalResourceMap.find(Val: descriptor);
252 if (varIt == canonicalResourceMap.end())
253 return {};
254 return varIt->second;
255}
256
257spirv::GlobalVariableOp ResourceAliasAnalysis::getCanonicalResource(
258 spirv::GlobalVariableOp varOp) const {
259 auto descriptorIt = descriptorMap.find(Val: varOp);
260 if (descriptorIt == descriptorMap.end())
261 return {};
262 return getCanonicalResource(descriptor: descriptorIt->second);
263}
264
265spirv::SPIRVType
266ResourceAliasAnalysis::getElementType(spirv::GlobalVariableOp varOp) const {
267 auto it = elementTypeMap.find(Val: varOp);
268 if (it == elementTypeMap.end())
269 return {};
270 return it->second;
271}
272
273void ResourceAliasAnalysis::recordIfUnifiable(
274 const Descriptor &descriptor, ArrayRef<spirv::GlobalVariableOp> resources) {
275 // Collect the element types for all resources in the current set.
276 SmallVector<spirv::SPIRVType> elementTypes;
277 for (spirv::GlobalVariableOp resource : resources) {
278 Type elementType = getRuntimeArrayElementType(type: resource.getType());
279 if (!elementType)
280 return; // Unexpected resource variable type.
281
282 auto type = cast<spirv::SPIRVType>(Val&: elementType);
283 if (!type.isScalarOrVector())
284 return; // Unexpected resource element type.
285
286 elementTypes.push_back(Elt: type);
287 }
288
289 std::optional<int> index = deduceCanonicalResource(types: elementTypes);
290 if (!index)
291 return;
292
293 // Update internal data structures for later use.
294 resourceMap[descriptor].assign(in_start: resources.begin(), in_end: resources.end());
295 canonicalResourceMap[descriptor] = resources[*index];
296 for (const auto &resource : llvm::enumerate(First&: resources)) {
297 descriptorMap[resource.value()] = descriptor;
298 elementTypeMap[resource.value()] = elementTypes[resource.index()];
299 }
300}
301
302//===----------------------------------------------------------------------===//
303// Patterns
304//===----------------------------------------------------------------------===//
305
306template <typename OpTy>
307class ConvertAliasResource : public OpConversionPattern<OpTy> {
308public:
309 ConvertAliasResource(const ResourceAliasAnalysis &analysis,
310 MLIRContext *context, PatternBenefit benefit = 1)
311 : OpConversionPattern<OpTy>(context, benefit), analysis(analysis) {}
312
313protected:
314 const ResourceAliasAnalysis &analysis;
315};
316
317struct ConvertVariable : public ConvertAliasResource<spirv::GlobalVariableOp> {
318 using ConvertAliasResource::ConvertAliasResource;
319
320 LogicalResult
321 matchAndRewrite(spirv::GlobalVariableOp varOp, OpAdaptor adaptor,
322 ConversionPatternRewriter &rewriter) const override {
323 // Just remove the aliased resource. Users will be rewritten to use the
324 // canonical one.
325 rewriter.eraseOp(op: varOp);
326 return success();
327 }
328};
329
330struct ConvertAddressOf : public ConvertAliasResource<spirv::AddressOfOp> {
331 using ConvertAliasResource::ConvertAliasResource;
332
333 LogicalResult
334 matchAndRewrite(spirv::AddressOfOp addressOp, OpAdaptor adaptor,
335 ConversionPatternRewriter &rewriter) const override {
336 // Rewrite the AddressOf op to get the address of the canoncical resource.
337 auto moduleOp = addressOp->getParentOfType<spirv::ModuleOp>();
338 auto srcVarOp = cast<spirv::GlobalVariableOp>(
339 Val: SymbolTable::lookupSymbolIn(op: moduleOp, symbol: addressOp.getVariable()));
340 auto dstVarOp = analysis.getCanonicalResource(varOp: srcVarOp);
341 rewriter.replaceOpWithNewOp<spirv::AddressOfOp>(op: addressOp, args&: dstVarOp);
342 return success();
343 }
344};
345
346struct ConvertAccessChain : public ConvertAliasResource<spirv::AccessChainOp> {
347 using ConvertAliasResource::ConvertAliasResource;
348
349 LogicalResult
350 matchAndRewrite(spirv::AccessChainOp acOp, OpAdaptor adaptor,
351 ConversionPatternRewriter &rewriter) const override {
352 auto addressOp = acOp.getBasePtr().getDefiningOp<spirv::AddressOfOp>();
353 if (!addressOp)
354 return rewriter.notifyMatchFailure(arg&: acOp, msg: "base ptr not addressof op");
355
356 auto moduleOp = acOp->getParentOfType<spirv::ModuleOp>();
357 auto srcVarOp = cast<spirv::GlobalVariableOp>(
358 Val: SymbolTable::lookupSymbolIn(op: moduleOp, symbol: addressOp.getVariable()));
359 auto dstVarOp = analysis.getCanonicalResource(varOp: srcVarOp);
360
361 spirv::SPIRVType srcElemType = analysis.getElementType(varOp: srcVarOp);
362 spirv::SPIRVType dstElemType = analysis.getElementType(varOp: dstVarOp);
363
364 if (srcElemType == dstElemType ||
365 areSameBitwidthScalarOrVectorType(a: srcElemType, b: dstElemType)) {
366 // We have the same bitwidth for source and destination element types.
367 // Thie indices keep the same.
368 rewriter.replaceOpWithNewOp<spirv::AccessChainOp>(
369 op: acOp, args: adaptor.getBasePtr(), args: adaptor.getIndices());
370 return success();
371 }
372
373 Location loc = acOp.getLoc();
374
375 if (srcElemType.isIntOrFloat() && isa<VectorType>(Val: dstElemType)) {
376 // The source indices are for a buffer with scalar element types. Rewrite
377 // them into a buffer with vector element types. We need to scale the last
378 // index for the vector as a whole, then add one level of index for inside
379 // the vector.
380 std::optional<int64_t> srcBytes = srcElemType.getSizeInBytes();
381 std::optional<int64_t> dstBytes = dstElemType.getSizeInBytes();
382 if (!srcBytes || !dstBytes)
383 return rewriter.notifyMatchFailure(arg&: acOp, msg: "unknown element byte size");
384 int srcNumBytes = *srcBytes;
385 int dstNumBytes = *dstBytes;
386 assert(dstNumBytes >= srcNumBytes && dstNumBytes % srcNumBytes == 0);
387
388 auto indices = llvm::to_vector<4>(Range: acOp.getIndices());
389 Value oldIndex = indices.back();
390 Type indexType = oldIndex.getType();
391
392 int ratio = dstNumBytes / srcNumBytes;
393 auto ratioValue = spirv::ConstantOp::create(
394 builder&: rewriter, location: loc, constant: indexType, value: rewriter.getIntegerAttr(type: indexType, value: ratio));
395
396 indices.back() =
397 spirv::SDivOp::create(builder&: rewriter, location: loc, result: indexType, operand1: oldIndex, operand2: ratioValue);
398 indices.push_back(Elt: spirv::SModOp::create(builder&: rewriter, location: loc, result: indexType,
399 operand1: oldIndex, operand2: ratioValue));
400
401 rewriter.replaceOpWithNewOp<spirv::AccessChainOp>(
402 op: acOp, args: adaptor.getBasePtr(), args&: indices);
403 return success();
404 }
405
406 if ((srcElemType.isIntOrFloat() && dstElemType.isIntOrFloat()) ||
407 (isa<VectorType>(Val: srcElemType) && isa<VectorType>(Val: dstElemType))) {
408 // The source indices are for a buffer with larger bitwidth scalar/vector
409 // element types. Rewrite them into a buffer with smaller bitwidth element
410 // types. We only need to scale the last index.
411 std::optional<int64_t> srcBytes = srcElemType.getSizeInBytes();
412 std::optional<int64_t> dstBytes = dstElemType.getSizeInBytes();
413 if (!srcBytes || !dstBytes)
414 return rewriter.notifyMatchFailure(arg&: acOp, msg: "unknown element byte size");
415 int srcNumBytes = *srcBytes;
416 int dstNumBytes = *dstBytes;
417 assert(srcNumBytes >= dstNumBytes && srcNumBytes % dstNumBytes == 0);
418
419 auto indices = llvm::to_vector<4>(Range: acOp.getIndices());
420 Value oldIndex = indices.back();
421 Type indexType = oldIndex.getType();
422
423 int ratio = srcNumBytes / dstNumBytes;
424 auto ratioValue = spirv::ConstantOp::create(
425 builder&: rewriter, location: loc, constant: indexType, value: rewriter.getIntegerAttr(type: indexType, value: ratio));
426
427 indices.back() =
428 spirv::IMulOp::create(builder&: rewriter, location: loc, result: indexType, operand1: oldIndex, operand2: ratioValue);
429
430 rewriter.replaceOpWithNewOp<spirv::AccessChainOp>(
431 op: acOp, args: adaptor.getBasePtr(), args&: indices);
432 return success();
433 }
434
435 return rewriter.notifyMatchFailure(
436 arg&: acOp, msg: "unsupported src/dst types for spirv.AccessChain");
437 }
438};
439
440struct ConvertLoad : public ConvertAliasResource<spirv::LoadOp> {
441 using ConvertAliasResource::ConvertAliasResource;
442
443 LogicalResult
444 matchAndRewrite(spirv::LoadOp loadOp, OpAdaptor adaptor,
445 ConversionPatternRewriter &rewriter) const override {
446 auto srcPtrType = cast<spirv::PointerType>(Val: loadOp.getPtr().getType());
447 auto srcElemType = cast<spirv::SPIRVType>(Val: srcPtrType.getPointeeType());
448 auto dstPtrType = cast<spirv::PointerType>(Val: adaptor.getPtr().getType());
449 auto dstElemType = cast<spirv::SPIRVType>(Val: dstPtrType.getPointeeType());
450
451 Location loc = loadOp.getLoc();
452 auto newLoadOp = spirv::LoadOp::create(builder&: rewriter, location: loc, basePtr: adaptor.getPtr());
453 if (srcElemType == dstElemType) {
454 rewriter.replaceOp(op: loadOp, newValues: newLoadOp->getResults());
455 return success();
456 }
457
458 if (areSameBitwidthScalarOrVectorType(a: srcElemType, b: dstElemType)) {
459 auto castOp = spirv::BitcastOp::create(builder&: rewriter, location: loc, result: srcElemType,
460 operand: newLoadOp.getValue());
461 rewriter.replaceOp(op: loadOp, newValues: castOp->getResults());
462
463 return success();
464 }
465
466 if ((srcElemType.isIntOrFloat() && dstElemType.isIntOrFloat()) ||
467 (isa<VectorType>(Val: srcElemType) && isa<VectorType>(Val: dstElemType))) {
468 // The source and destination have scalar types of different bitwidths, or
469 // vector types of different component counts. For such cases, we load
470 // multiple smaller bitwidth values and construct a larger bitwidth one.
471
472 std::optional<int64_t> srcBytes = srcElemType.getSizeInBytes();
473 std::optional<int64_t> dstBytes = dstElemType.getSizeInBytes();
474 if (!srcBytes || !dstBytes)
475 return rewriter.notifyMatchFailure(arg&: loadOp, msg: "unknown element byte size");
476 int srcNumBytes = *srcBytes;
477 int dstNumBytes = *dstBytes;
478 assert(srcNumBytes > dstNumBytes && srcNumBytes % dstNumBytes == 0);
479 int ratio = srcNumBytes / dstNumBytes;
480 if (ratio > 4)
481 return rewriter.notifyMatchFailure(arg&: loadOp, msg: "more than 4 components");
482
483 SmallVector<Value> components;
484 components.reserve(N: ratio);
485 components.push_back(Elt: newLoadOp);
486
487 auto acOp = adaptor.getPtr().getDefiningOp<spirv::AccessChainOp>();
488 if (!acOp)
489 return rewriter.notifyMatchFailure(arg&: loadOp, msg: "ptr not spirv.AccessChain");
490
491 auto i32Type = rewriter.getI32Type();
492 Value oneValue = spirv::ConstantOp::getOne(type: i32Type, loc, builder&: rewriter);
493 auto indices = llvm::to_vector<4>(Range: acOp.getIndices());
494 for (int i = 1; i < ratio; ++i) {
495 // Load all subsequent components belonging to this element.
496 indices.back() = spirv::IAddOp::create(builder&: rewriter, location: loc, result: i32Type,
497 operand1: indices.back(), operand2: oneValue);
498 auto componentAcOp = spirv::AccessChainOp::create(
499 builder&: rewriter, location: loc, basePtr: acOp.getBasePtr(), indices);
500 // Assuming little endian, this reads lower-ordered bits of the number
501 // to lower-numbered components of the vector.
502 components.push_back(
503 Elt: spirv::LoadOp::create(builder&: rewriter, location: loc, basePtr: componentAcOp));
504 }
505
506 // Create a vector of the components and then cast back to the larger
507 // bitwidth element type. For spirv.bitcast, the lower-numbered components
508 // of the vector map to lower-ordered bits of the larger bitwidth element
509 // type.
510
511 Type vectorType = srcElemType;
512 if (!isa<VectorType>(Val: srcElemType))
513 vectorType = VectorType::get(shape: {ratio}, elementType: dstElemType);
514
515 // If both the source and destination are vector types, we need to make
516 // sure the scalar type is the same for composite construction later.
517 if (auto srcElemVecType = dyn_cast<VectorType>(Val&: srcElemType))
518 if (auto dstElemVecType = dyn_cast<VectorType>(Val&: dstElemType)) {
519 if (srcElemVecType.getElementType() !=
520 dstElemVecType.getElementType()) {
521 int64_t count =
522 dstNumBytes / (srcElemVecType.getElementTypeBitWidth() / 8);
523
524 // Make sure not to create 1-element vectors, which are illegal in
525 // SPIR-V.
526 Type castType = srcElemVecType.getElementType();
527 if (count > 1)
528 castType = VectorType::get(shape: {count}, elementType: castType);
529
530 for (Value &c : components)
531 c = spirv::BitcastOp::create(builder&: rewriter, location: loc, result: castType, operand: c);
532 }
533 }
534 Value vectorValue = spirv::CompositeConstructOp::create(
535 builder&: rewriter, location: loc, result: vectorType, constituents: components);
536
537 if (!isa<VectorType>(Val: srcElemType))
538 vectorValue =
539 spirv::BitcastOp::create(builder&: rewriter, location: loc, result: srcElemType, operand: vectorValue);
540 rewriter.replaceOp(op: loadOp, newValues: vectorValue);
541 return success();
542 }
543
544 return rewriter.notifyMatchFailure(
545 arg&: loadOp, msg: "unsupported src/dst types for spirv.Load");
546 }
547};
548
549struct ConvertStore : public ConvertAliasResource<spirv::StoreOp> {
550 using ConvertAliasResource::ConvertAliasResource;
551
552 LogicalResult
553 matchAndRewrite(spirv::StoreOp storeOp, OpAdaptor adaptor,
554 ConversionPatternRewriter &rewriter) const override {
555 auto srcElemType =
556 cast<spirv::PointerType>(Val: storeOp.getPtr().getType()).getPointeeType();
557 auto dstElemType =
558 cast<spirv::PointerType>(Val: adaptor.getPtr().getType()).getPointeeType();
559 Location loc = storeOp.getLoc();
560 Value value = adaptor.getValue();
561 auto srcVector = dyn_cast<VectorType>(Val&: srcElemType);
562 auto dstVector = dyn_cast<VectorType>(Val&: dstElemType);
563 if (srcVector && dstVector && srcVector != dstVector) {
564 if (srcVector.getElementTypeBitWidth() !=
565 dstVector.getElementTypeBitWidth() ||
566 srcVector.getNumElements() % dstVector.getNumElements() != 0)
567 return rewriter.notifyMatchFailure(arg&: storeOp,
568 msg: "incompatible vector types");
569 auto acOp = adaptor.getPtr().getDefiningOp<spirv::AccessChainOp>();
570 if (!acOp)
571 return rewriter.notifyMatchFailure(arg&: storeOp,
572 msg: "ptr not spirv.AccessChain");
573 if (srcVector.getElementType() != dstVector.getElementType())
574 value = spirv::BitcastOp::create(
575 builder&: rewriter, location: loc,
576 result: VectorType::get(shape: srcVector.getShape(), elementType: dstVector.getElementType()),
577 operand: value);
578 unsigned count = dstVector.getNumElements();
579 unsigned ratio = srcVector.getNumElements() / count;
580 unsigned bytes = count * dstVector.getElementTypeBitWidth() / 8;
581
582 // Split the value into vectors matching the canonical resource and store
583 // them at consecutive vector indices. Adjust each store's alignment for
584 // its byte offset from the original pointer.
585 auto getPart = [&](unsigned i) -> Value {
586 if (ratio == 1)
587 return value;
588 SmallVector<int32_t> components;
589 for (unsigned j = 0; j < count; ++j)
590 components.push_back(Elt: i * count + j);
591 return spirv::VectorShuffleOp::create(
592 builder&: rewriter, location: loc, result: dstVector, vector1: value, vector2: value,
593 components: rewriter.getI32ArrayAttr(values: components));
594 };
595 auto firstStore = spirv::StoreOp::create(
596 builder&: rewriter, location: loc, ptr: adaptor.getPtr(), value: getPart(0),
597 memory_access: storeOp.getMemoryAccessAttr(), alignment: storeOp.getAlignmentAttr());
598 firstStore->setDiscardableAttrs(storeOp->getDiscardableAttrDictionary());
599 for (unsigned i = 1; i < ratio; ++i) {
600 Value part = getPart(i);
601 SmallVector<Value> indices = llvm::to_vector(Range: acOp.getIndices());
602 Type indexType = indices.back().getType();
603 auto offset = spirv::ConstantOp::create(
604 builder&: rewriter, location: loc, constant: indexType, value: rewriter.getIntegerAttr(type: indexType, value: i));
605 indices.back() = spirv::IAddOp::create(builder&: rewriter, location: loc, result: indexType,
606 operand1: indices.back(), operand2: offset);
607 Value ptr = spirv::AccessChainOp::create(builder&: rewriter, location: loc,
608 basePtr: acOp.getBasePtr(), indices);
609 IntegerAttr alignment = storeOp.getAlignmentAttr();
610 if (alignment)
611 alignment = rewriter.getI32IntegerAttr(
612 value: llvm::commonAlignment(A: llvm::Align(alignment.getInt()), Offset: i * bytes)
613 .value());
614 auto store = spirv::StoreOp::create(
615 builder&: rewriter, location: loc, ptr, value: part, memory_access: storeOp.getMemoryAccessAttr(), alignment);
616 store->setDiscardableAttrs(storeOp->getDiscardableAttrDictionary());
617 }
618 rewriter.eraseOp(op: storeOp);
619 return success();
620 }
621 if (srcElemType != dstElemType) {
622 if (!srcElemType.isIntOrFloat() || !dstElemType.isIntOrFloat())
623 return rewriter.notifyMatchFailure(arg&: storeOp, msg: "not scalar type");
624 if (!areSameBitwidthScalarOrVectorType(a: srcElemType, b: dstElemType))
625 return rewriter.notifyMatchFailure(arg&: storeOp, msg: "different bitwidth");
626 value = spirv::BitcastOp::create(builder&: rewriter, location: loc, result: dstElemType, operand: value);
627 }
628 auto newStore = rewriter.replaceOpWithNewOp<spirv::StoreOp>(
629 op: storeOp, args: adaptor.getPtr(), args&: value, args: storeOp.getMemoryAccessAttr(),
630 args: storeOp.getAlignmentAttr());
631 newStore->setDiscardableAttrs(storeOp->getDiscardableAttrDictionary());
632 return success();
633 }
634};
635
636//===----------------------------------------------------------------------===//
637// Pass
638//===----------------------------------------------------------------------===//
639
640namespace {
641class UnifyAliasedResourcePass final
642 : public spirv::impl::SPIRVUnifyAliasedResourcePassBase<
643 UnifyAliasedResourcePass> {
644public:
645 explicit UnifyAliasedResourcePass(spirv::GetTargetEnvFn getTargetEnv)
646 : getTargetEnvFn(std::move(getTargetEnv)) {}
647
648 void runOnOperation() override;
649
650private:
651 spirv::GetTargetEnvFn getTargetEnvFn;
652};
653
654void UnifyAliasedResourcePass::runOnOperation() {
655 spirv::ModuleOp moduleOp = getOperation();
656 MLIRContext *context = &getContext();
657
658 if (getTargetEnvFn) {
659 // This pass is only needed for targeting WebGPU, Metal, or layering
660 // Vulkan on Metal via MoltenVK, where we need to translate SPIR-V into
661 // WGSL or MSL. The translation has limitations.
662 spirv::TargetEnvAttr targetEnv = getTargetEnvFn(moduleOp);
663 spirv::ClientAPI clientAPI = targetEnv.getClientAPI();
664 bool isVulkanOnAppleDevices =
665 clientAPI == spirv::ClientAPI::Vulkan &&
666 targetEnv.getVendorID() == spirv::Vendor::Apple;
667 if (clientAPI != spirv::ClientAPI::WebGPU &&
668 clientAPI != spirv::ClientAPI::Metal && !isVulkanOnAppleDevices)
669 return;
670 }
671
672 // Analyze aliased resources first.
673 ResourceAliasAnalysis &analysis = getAnalysis<ResourceAliasAnalysis>();
674
675 ConversionTarget target(*context);
676 target.addDynamicallyLegalOp<spirv::GlobalVariableOp, spirv::AddressOfOp,
677 spirv::AccessChainOp, spirv::LoadOp,
678 spirv::StoreOp>(
679 callback: [&analysis](Operation *op) { return !analysis.shouldUnify(op); });
680 target.addLegalDialect<spirv::SPIRVDialect>();
681
682 // Run patterns to rewrite usages of non-canonical resources.
683 RewritePatternSet patterns(context);
684 patterns.add<ConvertVariable, ConvertAddressOf, ConvertAccessChain,
685 ConvertLoad, ConvertStore>(arg&: analysis, args&: context);
686 if (failed(Result: applyPartialConversion(op: moduleOp, target, patterns: std::move(patterns))))
687 return signalPassFailure();
688
689 // Drop aliased attribute if we only have one single bound resource for a
690 // descriptor. We need to re-collect the map here given in the above the
691 // conversion is best effort; certain sets may not be converted.
692 AliasedResourceMap resourceMap =
693 collectAliasedResources(moduleOp: cast<spirv::ModuleOp>(Val&: moduleOp));
694 for (const auto &dr : resourceMap) {
695 const auto &resources = dr.second;
696 if (resources.size() == 1)
697 resources.front()->removeDiscardableAttr(name: "aliased");
698 }
699}
700} // namespace
701
702std::unique_ptr<mlir::OperationPass<spirv::ModuleOp>>
703spirv::createUnifyAliasedResourcePass(spirv::GetTargetEnvFn getTargetEnv) {
704 return std::make_unique<UnifyAliasedResourcePass>(args: std::move(getTargetEnv));
705}
706