1//===- LowerABIAttributesPass.cpp - Decorate composite type ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements a pass to lower attributes that specify the shader ABI
10// for the functions in the generated SPIR-V module.
11//
12//===----------------------------------------------------------------------===//
13
14#include "mlir/Dialect/SPIRV/Transforms/Passes.h"
15
16#include "mlir/Dialect/SPIRV/IR/SPIRVAttributes.h"
17#include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
18#include "mlir/Dialect/SPIRV/IR/SPIRVEnums.h"
19#include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
20#include "mlir/Dialect/SPIRV/IR/TargetAndABI.h"
21#include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h"
22#include "mlir/Dialect/SPIRV/Utils/LayoutUtils.h"
23#include "mlir/IR/BuiltinAttributes.h"
24#include "mlir/Transforms/DialectConversion.h"
25#include "llvm/Support/FormatVariadic.h"
26
27namespace mlir {
28namespace spirv {
29#define GEN_PASS_DEF_SPIRVLOWERABIATTRIBUTESPASS
30#include "mlir/Dialect/SPIRV/Transforms/Passes.h.inc"
31} // namespace spirv
32} // namespace mlir
33
34using namespace mlir;
35
36/// Creates a global variable for an argument based on the ABI info.
37static spirv::GlobalVariableOp
38createGlobalVarForEntryPointArgument(OpBuilder &builder, spirv::FuncOp funcOp,
39 unsigned argIndex,
40 spirv::InterfaceVarABIAttr abiInfo) {
41 auto spirvModule = funcOp->getParentOfType<spirv::ModuleOp>();
42 if (!spirvModule)
43 return nullptr;
44
45 OpBuilder::InsertionGuard moduleInsertionGuard(builder);
46 builder.setInsertionPoint(funcOp.getOperation());
47 std::string varName =
48 funcOp.getName().str() + "_arg_" + std::to_string(val: argIndex);
49
50 // Get the type of variable. If this is a scalar/vector type and has an ABI
51 // info create a variable of type !spirv.ptr<!spirv.struct<elementType>>. If
52 // not it must already be a !spirv.ptr<!spirv.struct<...>>.
53 auto varType = funcOp.getFunctionType().getInput(i: argIndex);
54 if (cast<spirv::SPIRVType>(Val&: varType).isScalarOrVector()) {
55 auto storageClass = abiInfo.getStorageClass();
56 if (!storageClass)
57 return nullptr;
58 varType =
59 spirv::PointerType::get(pointeeType: spirv::StructType::get(memberTypes: varType), storageClass: *storageClass);
60 }
61 auto varPtrType = cast<spirv::PointerType>(Val&: varType);
62 Type pointeeType = varPtrType.getPointeeType();
63
64 // Images are an opaque type and so we can just return a pointer to an image.
65 // Note that currently only sampled images are supported in the SPIR-V
66 // lowering.
67 if (isa<spirv::SampledImageType>(Val: pointeeType))
68 return spirv::GlobalVariableOp::create(builder, location: funcOp.getLoc(), type: varType,
69 name: varName, descriptorSet: abiInfo.getDescriptorSet(),
70 binding: abiInfo.getBinding());
71
72 auto varPointeeType = cast<spirv::StructType>(Val&: pointeeType);
73
74 // Set the offset information.
75 varPointeeType =
76 cast<spirv::StructType>(Val: VulkanLayoutUtils::decorateType(structType: varPointeeType));
77
78 if (!varPointeeType)
79 return nullptr;
80
81 varType =
82 spirv::PointerType::get(pointeeType: varPointeeType, storageClass: varPtrType.getStorageClass());
83
84 return spirv::GlobalVariableOp::create(builder, location: funcOp.getLoc(), type: varType,
85 name: varName, descriptorSet: abiInfo.getDescriptorSet(),
86 binding: abiInfo.getBinding());
87}
88
89/// Creates a global variable for an argument or result based on the ABI info.
90static spirv::GlobalVariableOp
91createGlobalVarForGraphEntryPoint(OpBuilder &builder, spirv::GraphARMOp graphOp,
92 unsigned index, bool isArg,
93 spirv::InterfaceVarABIAttr abiInfo) {
94 auto spirvModule = graphOp->getParentOfType<spirv::ModuleOp>();
95 if (!spirvModule)
96 return nullptr;
97
98 OpBuilder::InsertionGuard moduleInsertionGuard(builder);
99 builder.setInsertionPoint(graphOp.getOperation());
100 std::string varName = llvm::formatv(Fmt: "{}_{}_{}", Vals: graphOp.getName(),
101 Vals: isArg ? "arg" : "res", Vals&: index);
102
103 Type varType = isArg ? graphOp.getFunctionType().getInput(i: index)
104 : graphOp.getFunctionType().getResult(i: index);
105
106 auto pointerType = spirv::PointerType::get(
107 pointeeType: varType,
108 storageClass: abiInfo.getStorageClass().value_or(u: spirv::StorageClass::UniformConstant));
109
110 return spirv::GlobalVariableOp::create(builder, location: graphOp.getLoc(), type: pointerType,
111 name: varName, descriptorSet: abiInfo.getDescriptorSet(),
112 binding: abiInfo.getBinding());
113}
114
115/// Gets the global variables that need to be specified as interface variable
116/// with an spirv.EntryPointOp. Traverses the body of a entry function to do so.
117static LogicalResult
118getInterfaceVariables(mlir::FunctionOpInterface funcOp,
119 SmallVectorImpl<Attribute> &interfaceVars) {
120 auto module = funcOp->getParentOfType<spirv::ModuleOp>();
121 if (!module) {
122 return failure();
123 }
124 spirv::TargetEnvAttr targetEnvAttr = spirv::lookupTargetEnv(op: funcOp);
125 spirv::TargetEnv targetEnv(targetEnvAttr);
126
127 SetVector<Operation *> interfaceVarSet;
128
129 // TODO: This should in reality traverse the entry function
130 // call graph and collect all the interfaces. For now, just traverse the
131 // instructions in this function.
132 funcOp.walk(callback: [&](spirv::AddressOfOp addressOfOp) {
133 auto var =
134 module.lookupSymbol<spirv::GlobalVariableOp>(name: addressOfOp.getVariable());
135 // Per SPIR-V spec: "Before version 1.4, the interface's
136 // storage classes are limited to the Input and Output storage classes.
137 // Starting with version 1.4, the interface's storage classes are all
138 // storage classes used in declaring all global variables referenced by the
139 // entry point’s call tree."
140 const spirv::StorageClass storageClass =
141 cast<spirv::PointerType>(Val: var.getType()).getStorageClass();
142 if ((targetEnvAttr && targetEnv.getVersion() >= spirv::Version::V_1_4) ||
143 (llvm::is_contained(
144 Set: {spirv::StorageClass::Input, spirv::StorageClass::Output},
145 Element: storageClass))) {
146 interfaceVarSet.insert(X: var.getOperation());
147 }
148 });
149 for (auto &var : interfaceVarSet) {
150 interfaceVars.push_back(Elt: SymbolRefAttr::get(
151 ctx: funcOp.getContext(), value: cast<spirv::GlobalVariableOp>(Val: var).getSymName()));
152 }
153 return success();
154}
155
156/// Lowers the entry point attribute.
157static LogicalResult lowerEntryPointABIAttr(spirv::FuncOp funcOp,
158 OpBuilder &builder) {
159 auto entryPointAttrName = spirv::getEntryPointABIAttrName();
160 auto entryPointAttr =
161 funcOp->getDiscardableAttrOfType<spirv::EntryPointABIAttr>(
162 name: entryPointAttrName);
163 if (!entryPointAttr) {
164 return failure();
165 }
166
167 spirv::TargetEnvAttr targetEnvAttr = spirv::lookupTargetEnv(op: funcOp);
168 spirv::TargetEnv targetEnv(targetEnvAttr);
169
170 OpBuilder::InsertionGuard moduleInsertionGuard(builder);
171 auto spirvModule = funcOp->getParentOfType<spirv::ModuleOp>();
172 builder.setInsertionPointToEnd(spirvModule.getBody());
173
174 // Adds the spirv.EntryPointOp after collecting all the interface variables
175 // needed.
176 SmallVector<Attribute, 1> interfaceVars;
177 if (failed(Result: getInterfaceVariables(funcOp, interfaceVars))) {
178 return failure();
179 }
180
181 FailureOr<spirv::ExecutionModel> executionModel =
182 spirv::getExecutionModel(targetAttr: targetEnvAttr);
183 if (failed(Result: executionModel))
184 return funcOp.emitRemark(message: "lower entry point failure: could not select "
185 "execution model based on 'spirv.target_env'");
186
187 spirv::EntryPointOp::create(builder, location: funcOp.getLoc(), executionModel: *executionModel, function: funcOp,
188 interfaceVars);
189
190 // Specifies the spirv.ExecutionModeOp.
191 if (DenseI32ArrayAttr workgroupSizeAttr = entryPointAttr.getWorkgroupSize()) {
192 std::optional<ArrayRef<spirv::Capability>> caps =
193 spirv::getCapabilities(value: spirv::ExecutionMode::LocalSize);
194 if (!caps || targetEnv.allows(*caps)) {
195 spirv::ExecutionModeOp::create(builder, location: funcOp.getLoc(), function: funcOp,
196 executionMode: spirv::ExecutionMode::LocalSize,
197 params: workgroupSizeAttr.asArrayRef());
198 // Erase workgroup size.
199 entryPointAttr = spirv::EntryPointABIAttr::get(
200 context: entryPointAttr.getContext(), workgroup_size: DenseI32ArrayAttr(),
201 subgroup_size: entryPointAttr.getSubgroupSize(), target_width: entryPointAttr.getTargetWidth());
202 }
203 }
204 if (std::optional<int> subgroupSize = entryPointAttr.getSubgroupSize()) {
205 std::optional<ArrayRef<spirv::Capability>> caps =
206 spirv::getCapabilities(value: spirv::ExecutionMode::SubgroupSize);
207 if (!caps || targetEnv.allows(*caps)) {
208 spirv::ExecutionModeOp::create(builder, location: funcOp.getLoc(), function: funcOp,
209 executionMode: spirv::ExecutionMode::SubgroupSize,
210 params: *subgroupSize);
211 // Erase subgroup size.
212 entryPointAttr = spirv::EntryPointABIAttr::get(
213 context: entryPointAttr.getContext(), workgroup_size: entryPointAttr.getWorkgroupSize(),
214 subgroup_size: std::nullopt, target_width: entryPointAttr.getTargetWidth());
215 }
216 }
217 if (std::optional<int> targetWidth = entryPointAttr.getTargetWidth()) {
218 std::optional<ArrayRef<spirv::Capability>> caps =
219 spirv::getCapabilities(value: spirv::ExecutionMode::SignedZeroInfNanPreserve);
220 if (!caps || targetEnv.allows(*caps)) {
221 spirv::ExecutionModeOp::create(
222 builder, location: funcOp.getLoc(), function: funcOp,
223 executionMode: spirv::ExecutionMode::SignedZeroInfNanPreserve, params: *targetWidth);
224 // Erase target width.
225 entryPointAttr = spirv::EntryPointABIAttr::get(
226 context: entryPointAttr.getContext(), workgroup_size: entryPointAttr.getWorkgroupSize(),
227 subgroup_size: entryPointAttr.getSubgroupSize(), target_width: std::nullopt);
228 }
229 }
230 if (entryPointAttr.getWorkgroupSize() || entryPointAttr.getSubgroupSize() ||
231 entryPointAttr.getTargetWidth())
232 funcOp->setDiscardableAttr(name: entryPointAttrName, value: entryPointAttr);
233 else
234 funcOp->removeDiscardableAttr(name: entryPointAttrName);
235 return success();
236}
237
238namespace {
239/// A pattern to convert function signature according to interface variable ABI
240/// attributes.
241///
242/// Specifically, this pattern creates global variables according to interface
243/// variable ABI attributes attached to function arguments and converts all
244/// function argument uses to those global variables. This is necessary because
245/// Vulkan requires all shader entry points to be of void(void) type.
246class ProcessInterfaceVarABI final : public OpConversionPattern<spirv::FuncOp> {
247public:
248 using Base::Base;
249
250 LogicalResult
251 matchAndRewrite(spirv::FuncOp funcOp, OpAdaptor adaptor,
252 ConversionPatternRewriter &rewriter) const override;
253};
254
255/// A pattern to convert graph signature according to interface variable ABI
256/// attributes.
257///
258/// Specifically, this pattern creates global variables according to interface
259/// variable ABI attributes attached to graph arguments and results.
260class ProcessGraphInterfaceVarABI final
261 : public OpConversionPattern<spirv::GraphARMOp> {
262public:
263 using Base::Base;
264
265 LogicalResult
266 matchAndRewrite(spirv::GraphARMOp graphOp, OpAdaptor adaptor,
267 ConversionPatternRewriter &rewriter) const override;
268};
269
270/// Pass to implement the ABI information specified as attributes.
271class LowerABIAttributesPass final
272 : public spirv::impl::SPIRVLowerABIAttributesPassBase<
273 LowerABIAttributesPass> {
274 void runOnOperation() override;
275};
276} // namespace
277
278LogicalResult ProcessInterfaceVarABI::matchAndRewrite(
279 spirv::FuncOp funcOp, OpAdaptor adaptor,
280 ConversionPatternRewriter &rewriter) const {
281 if (!funcOp->getDiscardableAttrOfType<spirv::EntryPointABIAttr>(
282 name: spirv::getEntryPointABIAttrName())) {
283 // TODO: Non-entry point functions are not handled.
284 return failure();
285 }
286 TypeConverter::SignatureConversion signatureConverter(
287 funcOp.getFunctionType().getNumInputs());
288
289 auto &typeConverter = *getTypeConverter<SPIRVTypeConverter>();
290 auto indexType = typeConverter.getIndexType();
291
292 auto attrName = spirv::getInterfaceVarABIAttrName();
293
294 OpBuilder::InsertionGuard funcInsertionGuard(rewriter);
295 rewriter.setInsertionPointToStart(&funcOp.front());
296
297 for (const auto &argType :
298 llvm::enumerate(First: funcOp.getFunctionType().getInputs())) {
299 auto abiInfo = funcOp.getArgAttrOfType<spirv::InterfaceVarABIAttr>(
300 index: argType.index(), name: attrName);
301 if (!abiInfo) {
302 // TODO: For non-entry point functions, it should be legal
303 // to pass around scalar/vector values and return a scalar/vector. For now
304 // non-entry point functions are not handled in this ABI lowering and will
305 // produce an error.
306 return failure();
307 }
308 spirv::GlobalVariableOp var = createGlobalVarForEntryPointArgument(
309 builder&: rewriter, funcOp, argIndex: argType.index(), abiInfo);
310 if (!var)
311 return failure();
312
313 // Insert spirv::AddressOf and spirv::AccessChain operations.
314 Value replacement =
315 spirv::AddressOfOp::create(builder&: rewriter, location: funcOp.getLoc(), var);
316 // Check if the arg is a scalar or vector type. In that case, the value
317 // needs to be loaded into registers.
318 // TODO: This is loading value of the scalar into registers
319 // at the start of the function. It is probably better to do the load just
320 // before the use. There might be multiple loads and currently there is no
321 // easy way to replace all uses with a sequence of operations.
322 if (cast<spirv::SPIRVType>(Val: argType.value()).isScalarOrVector()) {
323 auto zero =
324 spirv::ConstantOp::getZero(type: indexType, loc: funcOp.getLoc(), builder&: rewriter);
325 auto loadPtr = spirv::AccessChainOp::create(
326 builder&: rewriter, location: funcOp.getLoc(), basePtr: replacement, indices: zero.getConstant());
327 replacement = spirv::LoadOp::create(builder&: rewriter, location: funcOp.getLoc(), basePtr: loadPtr);
328 }
329 signatureConverter.remapInput(origInputNo: argType.index(), replacements: replacement);
330 }
331 if (failed(Result: rewriter.convertRegionTypes(region: &funcOp.getBody(), converter: *getTypeConverter(),
332 entryConversion: &signatureConverter)))
333 return failure();
334
335 // Creates a new function with the update signature.
336 rewriter.modifyOpInPlace(root: funcOp, callable: [&] {
337 funcOp.setType(
338 rewriter.getFunctionType(inputs: signatureConverter.getConvertedTypes(), results: {}));
339 });
340 return success();
341}
342
343LogicalResult ProcessGraphInterfaceVarABI::matchAndRewrite(
344 spirv::GraphARMOp graphOp, OpAdaptor adaptor,
345 ConversionPatternRewriter &rewriter) const {
346 // Non-entry point graphs are not handled.
347 if (!graphOp.getEntryPoint().value_or(u: false))
348 return failure();
349
350 TypeConverter::SignatureConversion signatureConverter(
351 graphOp.getFunctionType().getNumInputs());
352
353 StringRef attrName = spirv::getInterfaceVarABIAttrName();
354 SmallVector<Attribute, 4> interfaceVars;
355
356 // Convert arguments.
357 unsigned numInputs = graphOp.getFunctionType().getNumInputs();
358 unsigned numResults = graphOp.getFunctionType().getNumResults();
359 for (unsigned index = 0; index < numInputs; ++index) {
360 auto abiInfo =
361 graphOp.getArgAttrOfType<spirv::InterfaceVarABIAttr>(index, name: attrName);
362 if (!abiInfo)
363 return failure();
364 spirv::GlobalVariableOp var = createGlobalVarForGraphEntryPoint(
365 builder&: rewriter, graphOp, index, isArg: true, abiInfo);
366 if (!var)
367 return failure();
368 interfaceVars.push_back(
369 Elt: SymbolRefAttr::get(ctx: rewriter.getContext(), value: var.getSymName()));
370 }
371
372 for (unsigned index = 0; index < numResults; ++index) {
373 auto abiInfo = graphOp.getResultAttrOfType<spirv::InterfaceVarABIAttr>(
374 index, name: attrName);
375 if (!abiInfo)
376 return failure();
377 spirv::GlobalVariableOp var = createGlobalVarForGraphEntryPoint(
378 builder&: rewriter, graphOp, index, isArg: false, abiInfo);
379 if (!var)
380 return failure();
381 interfaceVars.push_back(
382 Elt: SymbolRefAttr::get(ctx: rewriter.getContext(), value: var.getSymName()));
383 }
384
385 // Update graph signature.
386 rewriter.modifyOpInPlace(root: graphOp, callable: [&] {
387 for (unsigned index = 0; index < numInputs; ++index) {
388 graphOp.removeArgAttr(index, name: attrName);
389 }
390 for (unsigned index = 0; index < numResults; ++index) {
391 graphOp.removeResultAttr(index, name: rewriter.getStringAttr(bytes: attrName));
392 }
393 });
394
395 spirv::GraphEntryPointARMOp::create(builder&: rewriter, location: graphOp.getLoc(), graph: graphOp,
396 interfaceVars);
397 return success();
398}
399
400void LowerABIAttributesPass::runOnOperation() {
401 // Uses the signature conversion methodology of the dialect conversion
402 // framework to implement the conversion.
403 spirv::ModuleOp module = getOperation();
404 MLIRContext *context = &getContext();
405
406 spirv::TargetEnvAttr targetEnvAttr = spirv::lookupTargetEnv(op: module);
407 if (!targetEnvAttr) {
408 module->emitOpError(message: "missing SPIR-V target env attribute");
409 return signalPassFailure();
410 }
411 spirv::TargetEnv targetEnv(targetEnvAttr);
412
413 SPIRVTypeConverter typeConverter(targetEnv);
414
415 // Insert a bitcast in the case of a pointer type change.
416 typeConverter.addSourceMaterialization(callback: [](OpBuilder &builder,
417 spirv::PointerType type,
418 ValueRange inputs, Location loc) {
419 if (inputs.size() != 1 || !isa<spirv::PointerType>(Val: inputs[0].getType()))
420 return Value();
421 return spirv::BitcastOp::create(builder, location: loc, result: type, operand: inputs[0]).getResult();
422 });
423
424 RewritePatternSet patterns(context);
425 patterns.add<ProcessInterfaceVarABI, ProcessGraphInterfaceVarABI>(
426 arg&: typeConverter, args&: context);
427
428 ConversionTarget target(*context);
429 // "Legal" function ops should have no interface variable ABI attributes.
430 target.addDynamicallyLegalOp<spirv::FuncOp>(callback: [&](spirv::FuncOp op) {
431 StringRef attrName = spirv::getInterfaceVarABIAttrName();
432 for (unsigned i = 0, e = op.getNumArguments(); i < e; ++i)
433 if (op.getArgAttr(index: i, name: attrName))
434 return false;
435 return true;
436 });
437 target.addDynamicallyLegalOp<spirv::GraphARMOp>(callback: [&](spirv::GraphARMOp op) {
438 StringRef attrName = spirv::getInterfaceVarABIAttrName();
439 for (unsigned i = 0, e = op.getNumArguments(); i < e; ++i)
440 if (op.getArgAttr(index: i, name: attrName))
441 return false;
442 for (unsigned i = 0, e = op.getNumResults(); i < e; ++i)
443 if (op.getResultAttr(index: i, name: attrName))
444 return false;
445 return true;
446 });
447
448 // All other SPIR-V ops are legal.
449 target.markUnknownOpDynamicallyLegal(fn: [](Operation *op) {
450 return op->getDialect()->getNamespace() ==
451 spirv::SPIRVDialect::getDialectNamespace();
452 });
453 if (failed(Result: applyPartialConversion(op: module, target, patterns: std::move(patterns))))
454 return signalPassFailure();
455
456 // Walks over all the FuncOps in spirv::ModuleOp to lower the entry point
457 // attributes.
458 OpBuilder builder(context);
459 SmallVector<spirv::FuncOp, 1> entryPointFns;
460 auto entryPointAttrName = spirv::getEntryPointABIAttrName();
461 module.walk(callback: [&](spirv::FuncOp funcOp) {
462 if (funcOp->getDiscardableAttrOfType<spirv::EntryPointABIAttr>(
463 name: entryPointAttrName)) {
464 entryPointFns.push_back(Elt: funcOp);
465 }
466 });
467 for (auto fn : entryPointFns) {
468 if (failed(Result: lowerEntryPointABIAttr(funcOp: fn, builder))) {
469 return signalPassFailure();
470 }
471 }
472}
473