1//===- LLVMDialect.cpp - MLIR SPIR-V dialect ------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the SPIR-V dialect in MLIR.
10//
11//===----------------------------------------------------------------------===//
12
13#include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
14
15#include "SPIRVParsingUtils.h"
16
17#include "mlir/Dialect/GPU/IR/CompilationInterfaces.h"
18#include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
19#include "mlir/Dialect/SPIRV/IR/SPIRVTypes.h"
20#include "mlir/Dialect/SPIRV/IR/TargetAndABI.h"
21#include "mlir/Dialect/UB/IR/UBOps.h"
22#include "mlir/IR/Builders.h"
23#include "mlir/IR/BuiltinTypes.h"
24#include "mlir/IR/DialectImplementation.h"
25#include "mlir/IR/MLIRContext.h"
26#include "mlir/Parser/Parser.h"
27#include "mlir/Transforms/InliningUtils.h"
28#include "llvm/ADT/Sequence.h"
29#include "llvm/ADT/StringExtras.h"
30#include "llvm/ADT/TypeSwitch.h"
31
32#include <optional>
33
34using namespace mlir;
35using namespace mlir::spirv;
36
37#include "mlir/Dialect/SPIRV/IR/SPIRVOpsDialect.cpp.inc"
38
39//===----------------------------------------------------------------------===//
40// InlinerInterface
41//===----------------------------------------------------------------------===//
42
43/// Returns true if the given region contains spirv.Return or spirv.ReturnValue
44/// ops.
45static inline bool containsReturn(Region &region) {
46 return llvm::any_of(Range&: region, P: [](Block &block) {
47 Operation *terminator = block.getTerminator();
48 return isa<spirv::ReturnOp, spirv::ReturnValueOp>(Val: terminator);
49 });
50}
51
52namespace {
53/// This class defines the interface for inlining within the SPIR-V dialect.
54struct SPIRVInlinerInterface : public DialectInlinerInterface {
55 using DialectInlinerInterface::DialectInlinerInterface;
56
57 /// All call operations within SPIRV can be inlined.
58 bool isLegalToInline(Operation *call, Operation *callable,
59 bool wouldBeCloned) const final {
60 return true;
61 }
62
63 /// Returns true if the given region 'src' can be inlined into the region
64 /// 'dest' that is attached to an operation registered to the current dialect.
65 bool isLegalToInline(Region *dest, Region *src, bool wouldBeCloned,
66 IRMapping &) const final {
67 // Return true here when inlining into spirv.func, spirv.mlir.selection, and
68 // spirv.mlir.loop operations.
69 auto *op = dest->getParentOp();
70 return isa<spirv::FuncOp, spirv::SelectionOp, spirv::LoopOp>(Val: op);
71 }
72
73 /// Returns true if the given operation 'op', that is registered to this
74 /// dialect, can be inlined into the region 'dest' that is attached to an
75 /// operation registered to the current dialect.
76 bool isLegalToInline(Operation *op, Region *dest, bool wouldBeCloned,
77 IRMapping &) const final {
78 // TODO: Enable inlining structured control flows with return.
79 if ((isa<spirv::SelectionOp, spirv::LoopOp>(Val: op)) &&
80 containsReturn(region&: op->getRegion(index: 0)))
81 return false;
82 // TODO: we need to filter OpKill here to avoid inlining it to
83 // a loop continue construct:
84 // https://github.com/KhronosGroup/SPIRV-Headers/issues/86
85 // For now, we just disallow inlining OpKill anywhere in the code,
86 // but this restriction should be relaxed, as pointed above.
87 if (isa<spirv::KillOp>(Val: op))
88 return false;
89
90 return true;
91 }
92
93 /// Handle the given inlined terminator by replacing it with a new operation
94 /// as necessary.
95 void handleTerminator(Operation *op, Block *newDest) const final {
96 if (auto returnOp = dyn_cast<spirv::ReturnOp>(Val: op)) {
97 auto builder = OpBuilder(op);
98 spirv::BranchOp::create(builder, location: op->getLoc(), successor: newDest);
99 op->erase();
100 } else if (auto retValOp = dyn_cast<spirv::ReturnValueOp>(Val: op)) {
101 auto builder = OpBuilder(op);
102 spirv::BranchOp::create(builder, location: retValOp->getLoc(), successor: newDest,
103 arguments: retValOp->getOperands());
104 op->erase();
105 }
106 }
107
108 /// Handle the given inlined terminator by replacing it with a new operation
109 /// as necessary.
110 void handleTerminator(Operation *op, ValueRange valuesToRepl) const final {
111 // Only spirv.ReturnValue needs to be handled here.
112 auto retValOp = dyn_cast<spirv::ReturnValueOp>(Val: op);
113 if (!retValOp)
114 return;
115
116 // Replace the values directly with the return operands.
117 assert(valuesToRepl.size() == 1 &&
118 "spirv.ReturnValue expected to only handle one result");
119 valuesToRepl.front().replaceAllUsesWith(newValue: retValOp.getValue());
120 }
121};
122} // namespace
123
124//===----------------------------------------------------------------------===//
125// SPIR-V Dialect
126//===----------------------------------------------------------------------===//
127
128void SPIRVDialect::initialize() {
129 registerAttributes();
130 registerTypes();
131
132 registerSPIRVDialectOperations(dialect: this);
133
134 addInterfaces<SPIRVInlinerInterface>();
135
136 // Allow unknown operations because SPIR-V is extensible.
137 allowUnknownOperations();
138 declarePromisedInterface<gpu::TargetAttrInterface, TargetEnvAttr>();
139}
140
141std::string SPIRVDialect::getAttributeName(Decoration decoration) {
142 return getDecorationString(decoration);
143}
144
145//===----------------------------------------------------------------------===//
146// Type Parsing
147//===----------------------------------------------------------------------===//
148
149// Forward declarations.
150template <typename ValTy>
151static std::optional<ValTy> parseAndVerify(SPIRVDialect const &dialect,
152 DialectAsmParser &parser);
153template <>
154std::optional<Type> parseAndVerify<Type>(SPIRVDialect const &dialect,
155 DialectAsmParser &parser);
156
157template <>
158std::optional<unsigned> parseAndVerify<unsigned>(SPIRVDialect const &dialect,
159 DialectAsmParser &parser);
160
161static Type parseAndVerifyType(SPIRVDialect const &dialect,
162 DialectAsmParser &parser) {
163 Type type;
164 SMLoc typeLoc = parser.getCurrentLocation();
165 if (parser.parseType(result&: type))
166 return Type();
167
168 // Allow SPIR-V dialect types.
169 if (&type.getDialect() == &dialect)
170 return type;
171
172 // Check other allowed types.
173 if (auto t = dyn_cast<FloatType>(Val&: type)) {
174 if (!ScalarType::isValid(t)) {
175 parser.emitError(loc: typeLoc,
176 message: "only 8/16/32/64-bit float type allowed but found ")
177 << type;
178 return Type();
179 }
180 } else if (auto t = dyn_cast<IntegerType>(Val&: type)) {
181 if (!ScalarType::isValid(t)) {
182 parser.emitError(loc: typeLoc,
183 message: "only 1/8/16/32/64-bit integer type allowed but found ")
184 << type;
185 return Type();
186 }
187 } else if (auto t = dyn_cast<VectorType>(Val&: type)) {
188 if (t.getRank() != 1) {
189 parser.emitError(loc: typeLoc, message: "only 1-D vector allowed but found ") << t;
190 return Type();
191 }
192 if (t.getNumElements() < 2) {
193 parser.emitError(loc: typeLoc, message: "SPIR-V does not allow one-element vectors");
194 return Type();
195 }
196 if (t.getNumElements() > 4) {
197 parser.emitError(
198 loc: typeLoc, message: "vector length has to be less than or equal to 4 but found ")
199 << t.getNumElements();
200 return Type();
201 }
202 if (!isa<ScalarType>(Val: t.getElementType())) {
203 parser.emitError(
204 loc: typeLoc,
205 message: "vector element type must be a SPIR-V scalar type but found ")
206 << t.getElementType();
207 return Type();
208 }
209 } else if (auto t = dyn_cast<TensorArmType>(Val&: type)) {
210 if (!isa<ScalarType>(Val: t.getElementType())) {
211 parser.emitError(
212 loc: typeLoc, message: "only scalar element type allowed in tensor type but found ")
213 << t.getElementType();
214 return Type();
215 }
216 } else {
217 parser.emitError(loc: typeLoc, message: "cannot use ")
218 << type << " to compose SPIR-V types";
219 return Type();
220 }
221
222 return type;
223}
224
225static Type parseAndVerifyMatrixType(SPIRVDialect const &dialect,
226 DialectAsmParser &parser) {
227 Type type;
228 SMLoc typeLoc = parser.getCurrentLocation();
229 if (parser.parseType(result&: type))
230 return Type();
231
232 if (auto t = dyn_cast<VectorType>(Val&: type)) {
233 if (t.getRank() != 1) {
234 parser.emitError(loc: typeLoc, message: "only 1-D vector allowed but found ") << t;
235 return Type();
236 }
237 if (t.getNumElements() > 4 || t.getNumElements() < 2) {
238 parser.emitError(loc: typeLoc,
239 message: "matrix columns size has to be less than or equal "
240 "to 4 and greater than or equal 2, but found ")
241 << t.getNumElements();
242 return Type();
243 }
244
245 if (!isa<FloatType>(Val: t.getElementType())) {
246 parser.emitError(loc: typeLoc, message: "matrix columns' elements must be of "
247 "Float type, got ")
248 << t.getElementType();
249 return Type();
250 }
251 } else {
252 parser.emitError(loc: typeLoc, message: "matrix must be composed using vector "
253 "type, got ")
254 << type;
255 return Type();
256 }
257
258 return type;
259}
260
261static Type parseAndVerifySampledImageType(SPIRVDialect const &dialect,
262 DialectAsmParser &parser) {
263 Type type;
264 SMLoc typeLoc = parser.getCurrentLocation();
265 if (parser.parseType(result&: type))
266 return Type();
267
268 auto imageType = dyn_cast<ImageType>(Val&: type);
269 if (!imageType) {
270 parser.emitError(loc: typeLoc,
271 message: "sampled image must be composed using image type, got ")
272 << type;
273 return Type();
274 }
275
276 if (llvm::is_contained(Set: {Dim::SubpassData, Dim::Buffer}, Element: imageType.getDim())) {
277 parser.emitError(
278 loc: typeLoc, message: "sampled image Dim must not be SubpassData or Buffer, got ")
279 << stringifyDim(imageType.getDim());
280 return Type();
281 }
282
283 return type;
284}
285
286/// Parses an optional `, stride = N` assembly segment. If no parsing failure
287/// occurs, writes `N` to `stride` if existing and writes 0 to `stride` if
288/// missing.
289static LogicalResult parseOptionalArrayStride(const SPIRVDialect &dialect,
290 DialectAsmParser &parser,
291 unsigned &stride) {
292 if (failed(Result: parser.parseOptionalComma())) {
293 stride = 0;
294 return success();
295 }
296
297 if (parser.parseKeyword(keyword: "stride") || parser.parseEqual())
298 return failure();
299
300 SMLoc strideLoc = parser.getCurrentLocation();
301 std::optional<unsigned> optStride = parseAndVerify<unsigned>(dialect, parser);
302 if (!optStride)
303 return failure();
304
305 if (!(stride = *optStride)) {
306 parser.emitError(loc: strideLoc, message: "ArrayStride must be greater than zero");
307 return failure();
308 }
309 return success();
310}
311
312// element-type ::= integer-type
313// | floating-point-type
314// | vector-type
315// | spirv-type
316//
317// array-type ::= `!spirv.array` `<` integer-literal `x` element-type
318// (`,` `stride` `=` integer-literal)? `>`
319static Type parseArrayType(SPIRVDialect const &dialect,
320 DialectAsmParser &parser) {
321 if (parser.parseLess())
322 return Type();
323
324 SmallVector<int64_t, 1> countDims;
325 SMLoc countLoc = parser.getCurrentLocation();
326 if (parser.parseDimensionList(dimensions&: countDims, /*allowDynamic=*/false))
327 return Type();
328 if (countDims.size() != 1) {
329 parser.emitError(loc: countLoc,
330 message: "expected single integer for array element count");
331 return Type();
332 }
333
334 // According to the SPIR-V spec:
335 // "Length is the number of elements in the array. It must be at least 1."
336 int64_t count = countDims[0];
337 if (count == 0) {
338 parser.emitError(loc: countLoc, message: "expected array length greater than 0");
339 return Type();
340 }
341
342 Type elementType = parseAndVerifyType(dialect, parser);
343 if (!elementType)
344 return Type();
345
346 unsigned stride = 0;
347 if (failed(Result: parseOptionalArrayStride(dialect, parser, stride)))
348 return Type();
349
350 if (parser.parseGreater())
351 return Type();
352 return ArrayType::get(elementType, elementCount: count, stride);
353}
354
355// cooperative-matrix-type ::=
356// `!spirv.coopmatrix` `<` rows `x` columns `x` element-type `,`
357// scope `,` use `>`
358static Type parseCooperativeMatrixType(SPIRVDialect const &dialect,
359 DialectAsmParser &parser) {
360 if (parser.parseLess())
361 return {};
362
363 SmallVector<int64_t, 2> dims;
364 SMLoc countLoc = parser.getCurrentLocation();
365 if (parser.parseDimensionList(dimensions&: dims, /*allowDynamic=*/false))
366 return {};
367
368 if (dims.size() != 2) {
369 parser.emitError(loc: countLoc, message: "expected row and column count");
370 return {};
371 }
372
373 auto elementTy = parseAndVerifyType(dialect, parser);
374 if (!elementTy)
375 return {};
376
377 Scope scope;
378 if (parser.parseComma() ||
379 spirv::parseEnumKeywordAttr(value&: scope, parser, attrName: "scope <id>"))
380 return {};
381
382 CooperativeMatrixUseKHR use;
383 if (parser.parseComma() ||
384 spirv::parseEnumKeywordAttr(value&: use, parser, attrName: "use <id>"))
385 return {};
386
387 if (parser.parseGreater())
388 return {};
389
390 return CooperativeMatrixType::get(elementType: elementTy, rows: dims[0], columns: dims[1], scope, use);
391}
392
393// tensor-arm-type ::=
394// `!spirv.arm.tensor` `<` dim0 `x` dim1 `x` ... `x` dimN `x` element-type`>`
395static Type parseTensorArmType(SPIRVDialect const &dialect,
396 DialectAsmParser &parser) {
397 if (parser.parseLess())
398 return {};
399
400 bool unranked = false;
401 SmallVector<int64_t, 4> dims;
402 SMLoc countLoc = parser.getCurrentLocation();
403
404 if (parser.parseOptionalStar().succeeded()) {
405 unranked = true;
406 if (parser.parseXInDimensionList())
407 return {};
408 } else if (parser.parseDimensionList(dimensions&: dims, /*allowDynamic=*/true)) {
409 return {};
410 }
411
412 if (!unranked && dims.empty()) {
413 parser.emitError(loc: countLoc, message: "arm.tensors do not support rank zero");
414 return {};
415 }
416
417 if (llvm::is_contained(Range&: dims, Element: 0)) {
418 parser.emitError(loc: countLoc, message: "arm.tensors do not support zero dimensions");
419 return {};
420 }
421
422 if (llvm::any_of(Range&: dims, P: [](int64_t dim) { return dim < 0; }) &&
423 llvm::any_of(Range&: dims, P: [](int64_t dim) { return dim > 0; })) {
424 parser.emitError(loc: countLoc, message: "arm.tensor shape dimensions must be either "
425 "fully dynamic or completed shaped");
426 return {};
427 }
428
429 auto elementTy = parseAndVerifyType(dialect, parser);
430 if (!elementTy)
431 return {};
432
433 if (parser.parseGreater())
434 return {};
435
436 return TensorArmType::get(shape: dims, elementType: elementTy);
437}
438
439// TODO: Reorder methods to be utilities first and parse*Type
440// methods in alphabetical order
441//
442// storage-class ::= `UniformConstant`
443// | `Uniform`
444// | `Workgroup`
445// | <and other storage classes...>
446//
447// pointer-type ::= `!spirv.ptr<` element-type `,` storage-class `>`
448static Type parsePointerType(SPIRVDialect const &dialect,
449 DialectAsmParser &parser) {
450 if (parser.parseLess())
451 return Type();
452
453 auto pointeeType = parseAndVerifyType(dialect, parser);
454 if (!pointeeType)
455 return Type();
456
457 StringRef storageClassSpec;
458 SMLoc storageClassLoc = parser.getCurrentLocation();
459 if (parser.parseComma() || parser.parseKeyword(keyword: &storageClassSpec))
460 return Type();
461
462 auto storageClass = symbolizeStorageClass(storageClassSpec);
463 if (!storageClass) {
464 parser.emitError(loc: storageClassLoc, message: "unknown storage class: ")
465 << storageClassSpec;
466 return Type();
467 }
468 if (parser.parseGreater())
469 return Type();
470 return PointerType::get(pointeeType, storageClass: *storageClass);
471}
472
473// runtime-array-type ::= `!spirv.rtarray` `<` element-type
474// (`,` `stride` `=` integer-literal)? `>`
475static Type parseRuntimeArrayType(SPIRVDialect const &dialect,
476 DialectAsmParser &parser) {
477 if (parser.parseLess())
478 return Type();
479
480 Type elementType = parseAndVerifyType(dialect, parser);
481 if (!elementType)
482 return Type();
483
484 unsigned stride = 0;
485 if (failed(Result: parseOptionalArrayStride(dialect, parser, stride)))
486 return Type();
487
488 if (parser.parseGreater())
489 return Type();
490 return RuntimeArrayType::get(elementType, stride);
491}
492
493// matrix-type ::= `!spirv.matrix` `<` integer-literal `x` element-type `>`
494static Type parseMatrixType(SPIRVDialect const &dialect,
495 DialectAsmParser &parser) {
496 if (parser.parseLess())
497 return Type();
498
499 SmallVector<int64_t, 1> countDims;
500 SMLoc countLoc = parser.getCurrentLocation();
501 if (parser.parseDimensionList(dimensions&: countDims, /*allowDynamic=*/false))
502 return Type();
503 if (countDims.size() != 1) {
504 parser.emitError(loc: countLoc, message: "expected single unsigned "
505 "integer for number of columns");
506 return Type();
507 }
508
509 int64_t columnCount = countDims[0];
510 // According to the specification, Matrices can have 2, 3, or 4 columns
511 if (columnCount < 2 || columnCount > 4) {
512 parser.emitError(loc: countLoc, message: "matrix is expected to have 2, 3, or 4 "
513 "columns");
514 return Type();
515 }
516
517 Type columnType = parseAndVerifyMatrixType(dialect, parser);
518 if (!columnType)
519 return Type();
520
521 if (parser.parseGreater())
522 return Type();
523
524 return MatrixType::get(columnType, columnCount);
525}
526
527// Specialize this function to parse each of the parameters that define an
528// ImageType. By default it assumes this is an enum type.
529template <typename ValTy>
530static std::optional<ValTy> parseAndVerify(SPIRVDialect const &dialect,
531 DialectAsmParser &parser) {
532 StringRef enumSpec;
533 SMLoc enumLoc = parser.getCurrentLocation();
534 if (parser.parseKeyword(keyword: &enumSpec)) {
535 return std::nullopt;
536 }
537
538 auto val = spirv::symbolizeEnum<ValTy>(enumSpec);
539 if (!val)
540 parser.emitError(loc: enumLoc, message: "unknown attribute: '") << enumSpec << "'";
541 return val;
542}
543
544template <>
545std::optional<Type> parseAndVerify<Type>(SPIRVDialect const &dialect,
546 DialectAsmParser &parser) {
547 // TODO: Further verify that the element type can be sampled
548 auto ty = parseAndVerifyType(dialect, parser);
549 if (!ty)
550 return std::nullopt;
551 return ty;
552}
553
554template <typename IntTy>
555static std::optional<IntTy> parseAndVerifyInteger(SPIRVDialect const &dialect,
556 DialectAsmParser &parser) {
557 IntTy offsetVal = std::numeric_limits<IntTy>::max();
558 if (parser.parseInteger(offsetVal))
559 return std::nullopt;
560 return offsetVal;
561}
562
563template <>
564std::optional<unsigned> parseAndVerify<unsigned>(SPIRVDialect const &dialect,
565 DialectAsmParser &parser) {
566 return parseAndVerifyInteger<unsigned>(dialect, parser);
567}
568
569namespace {
570// Functor object to parse a comma separated list of specs. The function
571// parseAndVerify does the actual parsing and verification of individual
572// elements. This is a functor since parsing the last element of the list
573// (termination condition) needs partial specialization.
574template <typename ParseType, typename... Args>
575struct ParseCommaSeparatedList {
576 std::optional<std::tuple<ParseType, Args...>>
577 operator()(SPIRVDialect const &dialect, DialectAsmParser &parser) const {
578 auto parseVal = parseAndVerify<ParseType>(dialect, parser);
579 if (!parseVal)
580 return std::nullopt;
581
582 auto numArgs = std::tuple_size<std::tuple<Args...>>::value;
583 if (numArgs != 0 && failed(Result: parser.parseComma()))
584 return std::nullopt;
585 auto remainingValues = ParseCommaSeparatedList<Args...>{}(dialect, parser);
586 if (!remainingValues)
587 return std::nullopt;
588 return std::tuple_cat(std::tuple<ParseType>(parseVal.value()),
589 remainingValues.value());
590 }
591};
592
593// Partial specialization of the function to parse a comma separated list of
594// specs to parse the last element of the list.
595template <typename ParseType>
596struct ParseCommaSeparatedList<ParseType> {
597 std::optional<std::tuple<ParseType>>
598 operator()(SPIRVDialect const &dialect, DialectAsmParser &parser) const {
599 if (auto value = parseAndVerify<ParseType>(dialect, parser))
600 return std::tuple<ParseType>(*value);
601 return std::nullopt;
602 }
603};
604} // namespace
605
606// dim ::= `1D` | `2D` | `3D` | `Cube` | <and other SPIR-V Dim specifiers...>
607//
608// depth-info ::= `NoDepth` | `IsDepth` | `DepthUnknown`
609//
610// arrayed-info ::= `NonArrayed` | `Arrayed`
611//
612// sampling-info ::= `SingleSampled` | `MultiSampled`
613//
614// sampler-use-info ::= `SamplerUnknown` | `NeedSampler` | `NoSampler`
615//
616// format ::= `Unknown` | `Rgba32f` | <and other SPIR-V Image formats...>
617//
618// image-type ::= `!spirv.image<` element-type `,` dim `,` depth-info `,`
619// arrayed-info `,` sampling-info `,`
620// sampler-use-info `,` format `>`
621static Type parseImageType(SPIRVDialect const &dialect,
622 DialectAsmParser &parser) {
623 if (parser.parseLess())
624 return Type();
625
626 auto value =
627 ParseCommaSeparatedList<Type, Dim, ImageDepthInfo, ImageArrayedInfo,
628 ImageSamplingInfo, ImageSamplerUseInfo,
629 ImageFormat>{}(dialect, parser);
630 if (!value)
631 return Type();
632
633 if (parser.parseGreater())
634 return Type();
635 return ImageType::get(*value);
636}
637
638// sampledImage-type :: = `!spirv.sampledImage<` image-type `>`
639static Type parseSampledImageType(SPIRVDialect const &dialect,
640 DialectAsmParser &parser) {
641 if (parser.parseLess())
642 return Type();
643
644 Type parsedType = parseAndVerifySampledImageType(dialect, parser);
645 if (!parsedType)
646 return Type();
647
648 if (parser.parseGreater())
649 return Type();
650 return SampledImageType::get(imageType: parsedType);
651}
652
653// Parse decorations associated with a member.
654static ParseResult parseStructMemberDecorations(
655 SPIRVDialect const &dialect, DialectAsmParser &parser,
656 ArrayRef<Type> memberTypes,
657 SmallVectorImpl<StructType::OffsetInfo> &offsetInfo,
658 SmallVectorImpl<StructType::MemberDecorationInfo> &memberDecorationInfo) {
659
660 // Check if the first element is offset.
661 SMLoc offsetLoc = parser.getCurrentLocation();
662 StructType::OffsetInfo offset = 0;
663 OptionalParseResult offsetParseResult = parser.parseOptionalInteger(result&: offset);
664 if (offsetParseResult.has_value()) {
665 if (failed(Result: *offsetParseResult))
666 return failure();
667
668 if (offsetInfo.size() != memberTypes.size() - 1) {
669 return parser.emitError(loc: offsetLoc,
670 message: "offset specification must be given for "
671 "all members");
672 }
673 offsetInfo.push_back(Elt: offset);
674 }
675
676 // Check for no spirv::Decorations.
677 if (succeeded(Result: parser.parseOptionalRSquare()))
678 return success();
679
680 // If there was an offset, make sure to parse the comma.
681 if (offsetParseResult.has_value() && parser.parseComma())
682 return failure();
683
684 // Check for spirv::Decorations.
685 auto parseDecorations = [&]() {
686 auto memberDecoration = parseAndVerify<spirv::Decoration>(dialect, parser);
687 if (!memberDecoration)
688 return failure();
689
690 // Parse member decoration value if it exists.
691 if (succeeded(Result: parser.parseOptionalEqual())) {
692 Attribute memberDecorationValue;
693 if (failed(Result: parser.parseAttribute(result&: memberDecorationValue)))
694 return failure();
695
696 memberDecorationInfo.emplace_back(
697 Args: static_cast<uint32_t>(memberTypes.size() - 1),
698 Args&: memberDecoration.value(), Args&: memberDecorationValue);
699 } else {
700 memberDecorationInfo.emplace_back(
701 Args: static_cast<uint32_t>(memberTypes.size() - 1),
702 Args&: memberDecoration.value(), Args: UnitAttr::get(context: dialect.getContext()));
703 }
704 return success();
705 };
706 if (failed(Result: parser.parseCommaSeparatedList(parseElementFn: parseDecorations)) ||
707 failed(Result: parser.parseRSquare()))
708 return failure();
709
710 return success();
711}
712
713// struct-member-decoration ::= integer-literal? spirv-decoration*
714// struct-type ::=
715// `!spirv.struct<` (id `,`)?
716// `(`
717// (spirv-type (`[` struct-member-decoration `]`)?)*
718// `)`
719// (`,` struct-decoration)?
720// `>`
721static Type parseStructType(SPIRVDialect const &dialect,
722 DialectAsmParser &parser) {
723 // TODO: This function is quite lengthy. Break it down into smaller chunks.
724
725 if (parser.parseLess())
726 return Type();
727
728 StringRef identifier;
729 FailureOr<DialectAsmParser::CyclicParseReset> cyclicParse;
730
731 // Check if this is an identified struct type.
732 if (succeeded(Result: parser.parseOptionalKeyword(keyword: &identifier))) {
733 // Check if this is a possible recursive reference.
734 auto structType =
735 StructType::getIdentified(context: dialect.getContext(), identifier);
736 cyclicParse = parser.tryStartCyclicParse(attrOrType: structType);
737 if (succeeded(Result: parser.parseOptionalGreater())) {
738 if (succeeded(Result: cyclicParse)) {
739 parser.emitError(
740 loc: parser.getNameLoc(),
741 message: "recursive struct reference not nested in struct definition");
742
743 return Type();
744 }
745
746 return structType;
747 }
748
749 if (failed(Result: parser.parseComma()))
750 return Type();
751
752 if (failed(Result: cyclicParse)) {
753 parser.emitError(loc: parser.getNameLoc(),
754 message: "identifier already used for an enclosing struct");
755 return Type();
756 }
757 }
758
759 if (failed(Result: parser.parseLParen()))
760 return Type();
761
762 if (succeeded(Result: parser.parseOptionalRParen()) &&
763 succeeded(Result: parser.parseOptionalGreater())) {
764 return StructType::getEmpty(context: dialect.getContext(), identifier);
765 }
766
767 StructType idStructTy;
768
769 if (!identifier.empty())
770 idStructTy = StructType::getIdentified(context: dialect.getContext(), identifier);
771
772 SmallVector<Type, 4> memberTypes;
773 SmallVector<StructType::OffsetInfo, 4> offsetInfo;
774 SmallVector<StructType::MemberDecorationInfo, 4> memberDecorationInfo;
775
776 do {
777 Type memberType;
778 if (parser.parseType(result&: memberType))
779 return Type();
780 if (!isa<SPIRVType>(Val: memberType)) {
781 parser.emitError(loc: parser.getNameLoc(),
782 message: "member type must be a valid SPIR-V type");
783 return Type();
784 }
785 memberTypes.push_back(Elt: memberType);
786
787 if (succeeded(Result: parser.parseOptionalLSquare()))
788 if (parseStructMemberDecorations(dialect, parser, memberTypes, offsetInfo,
789 memberDecorationInfo))
790 return Type();
791 } while (succeeded(Result: parser.parseOptionalComma()));
792
793 if (!offsetInfo.empty() && memberTypes.size() != offsetInfo.size()) {
794 parser.emitError(loc: parser.getNameLoc(),
795 message: "offset specification must be given for all members");
796 return Type();
797 }
798
799 if (failed(Result: parser.parseRParen()))
800 return Type();
801
802 SmallVector<StructType::StructDecorationInfo, 1> structDecorationInfo;
803
804 auto parseStructDecoration = [&]() {
805 std::optional<spirv::Decoration> decoration =
806 parseAndVerify<spirv::Decoration>(dialect, parser);
807 if (!decoration)
808 return failure();
809
810 // Parse decoration value if it exists.
811 if (succeeded(Result: parser.parseOptionalEqual())) {
812 Attribute decorationValue;
813 if (failed(Result: parser.parseAttribute(result&: decorationValue)))
814 return failure();
815
816 structDecorationInfo.emplace_back(Args&: decoration.value(), Args&: decorationValue);
817 } else {
818 structDecorationInfo.emplace_back(Args&: decoration.value(),
819 Args: UnitAttr::get(context: dialect.getContext()));
820 }
821 return success();
822 };
823
824 while (succeeded(Result: parser.parseOptionalComma()))
825 if (failed(Result: parseStructDecoration()))
826 return Type();
827
828 if (failed(Result: parser.parseGreater()))
829 return Type();
830
831 if (!identifier.empty()) {
832 if (failed(Result: idStructTy.trySetBody(memberTypes, offsetInfo,
833 memberDecorations: memberDecorationInfo,
834 structDecorations: structDecorationInfo)))
835 return Type();
836 return idStructTy;
837 }
838
839 return StructType::get(memberTypes, offsetInfo, memberDecorations: memberDecorationInfo,
840 structDecorations: structDecorationInfo);
841}
842
843// spirv-type ::= array-type
844// | element-type
845// | image-type
846// | pointer-type
847// | runtime-array-type
848// | sampled-image-type
849// | struct-type
850Type SPIRVDialect::parseType(DialectAsmParser &parser) const {
851 StringRef keyword;
852 if (parser.parseKeyword(keyword: &keyword))
853 return Type();
854
855 if (keyword == "array")
856 return parseArrayType(dialect: *this, parser);
857 if (keyword == "coopmatrix")
858 return parseCooperativeMatrixType(dialect: *this, parser);
859 if (keyword == "image")
860 return parseImageType(dialect: *this, parser);
861 if (keyword == "ptr")
862 return parsePointerType(dialect: *this, parser);
863 if (keyword == "rtarray")
864 return parseRuntimeArrayType(dialect: *this, parser);
865 if (keyword == "sampled_image")
866 return parseSampledImageType(dialect: *this, parser);
867 if (keyword == "sampler")
868 return SamplerType::get(context: getContext());
869 if (keyword == "named_barrier")
870 return NamedBarrierType::get(context: getContext());
871 if (keyword == "struct")
872 return parseStructType(dialect: *this, parser);
873 if (keyword == "matrix")
874 return parseMatrixType(dialect: *this, parser);
875 if (keyword == "arm.tensor")
876 return parseTensorArmType(dialect: *this, parser);
877 parser.emitError(loc: parser.getNameLoc(), message: "unknown SPIR-V type: ") << keyword;
878 return Type();
879}
880
881//===----------------------------------------------------------------------===//
882// Type Printing
883//===----------------------------------------------------------------------===//
884
885static void print(ArrayType type, DialectAsmPrinter &os) {
886 os << "array<" << type.getNumElements() << " x " << type.getElementType();
887 if (unsigned stride = type.getArrayStride())
888 os << ", stride=" << stride;
889 os << ">";
890}
891
892static void print(RuntimeArrayType type, DialectAsmPrinter &os) {
893 os << "rtarray<" << type.getElementType();
894 if (unsigned stride = type.getArrayStride())
895 os << ", stride=" << stride;
896 os << ">";
897}
898
899static void print(PointerType type, DialectAsmPrinter &os) {
900 os << "ptr<" << type.getPointeeType() << ", "
901 << stringifyStorageClass(type.getStorageClass()) << ">";
902}
903
904static void print(ImageType type, DialectAsmPrinter &os) {
905 os << "image<" << type.getElementType() << ", " << stringifyDim(type.getDim())
906 << ", " << stringifyImageDepthInfo(type.getDepthInfo()) << ", "
907 << stringifyImageArrayedInfo(type.getArrayedInfo()) << ", "
908 << stringifyImageSamplingInfo(type.getSamplingInfo()) << ", "
909 << stringifyImageSamplerUseInfo(type.getSamplerUseInfo()) << ", "
910 << stringifyImageFormat(type.getImageFormat()) << ">";
911}
912
913static void print(SampledImageType type, DialectAsmPrinter &os) {
914 os << "sampled_image<" << type.getImageType() << ">";
915}
916
917static void print(SamplerType type, DialectAsmPrinter &os) { os << "sampler"; }
918
919static void print(NamedBarrierType type, DialectAsmPrinter &os) {
920 os << "named_barrier";
921}
922
923static void print(StructType type, DialectAsmPrinter &os) {
924 FailureOr<AsmPrinter::CyclicPrintReset> cyclicPrint;
925
926 os << "struct<";
927
928 if (type.isIdentified()) {
929 os << type.getIdentifier();
930
931 cyclicPrint = os.tryStartCyclicPrint(attrOrType: type);
932 if (failed(Result: cyclicPrint)) {
933 os << ">";
934 return;
935 }
936
937 os << ", ";
938 }
939
940 os << "(";
941
942 auto printMember = [&](unsigned i) {
943 os << type.getElementType(i);
944 SmallVector<spirv::StructType::MemberDecorationInfo, 0> decorations;
945 type.getMemberDecorations(i, decorationsInfo&: decorations);
946 if (type.hasOffset() || !decorations.empty()) {
947 os << " [";
948 if (type.hasOffset()) {
949 os << type.getMemberOffset(i);
950 if (!decorations.empty())
951 os << ", ";
952 }
953 auto eachFn = [&os](spirv::StructType::MemberDecorationInfo decoration) {
954 os << stringifyDecoration(decoration.decoration);
955 if (decoration.hasValue()) {
956 os << "=";
957 os.printAttributeWithoutType(attr: decoration.decorationValue);
958 }
959 };
960 llvm::interleaveComma(c: decorations, os, each_fn: eachFn);
961 os << "]";
962 }
963 };
964 llvm::interleaveComma(c: llvm::seq<unsigned>(Begin: 0, End: type.getNumElements()), os,
965 each_fn: printMember);
966 os << ")";
967
968 SmallVector<spirv::StructType::StructDecorationInfo, 1> decorations;
969 type.getStructDecorations(structDecorations&: decorations);
970 if (!decorations.empty()) {
971 os << ", ";
972 auto eachFn = [&os](spirv::StructType::StructDecorationInfo decoration) {
973 os << stringifyDecoration(decoration.decoration);
974 if (decoration.hasValue()) {
975 os << "=";
976 os.printAttributeWithoutType(attr: decoration.decorationValue);
977 }
978 };
979 llvm::interleaveComma(c: decorations, os, each_fn: eachFn);
980 }
981
982 os << ">";
983}
984
985static void print(CooperativeMatrixType type, DialectAsmPrinter &os) {
986 os << "coopmatrix<" << type.getRows() << "x" << type.getColumns() << "x"
987 << type.getElementType() << ", " << type.getScope() << ", "
988 << type.getUse() << ">";
989}
990
991static void print(MatrixType type, DialectAsmPrinter &os) {
992 os << "matrix<" << type.getNumColumns() << " x " << type.getColumnType();
993 os << ">";
994}
995
996static void print(TensorArmType type, DialectAsmPrinter &os) {
997 os << "arm.tensor<";
998
999 llvm::interleave(
1000 c: type.getShape(), os,
1001 each_fn: [&](int64_t dim) {
1002 if (ShapedType::isDynamic(dValue: dim))
1003 os << '?';
1004 else
1005 os << dim;
1006 },
1007 separator: "x");
1008 if (!type.hasRank()) {
1009 os << "*";
1010 }
1011 os << "x" << type.getElementType() << ">";
1012}
1013
1014void SPIRVDialect::printType(Type type, DialectAsmPrinter &os) const {
1015 TypeSwitch<Type>(type)
1016 .Case<ArrayType, CooperativeMatrixType, PointerType, RuntimeArrayType,
1017 ImageType, SampledImageType, SamplerType, NamedBarrierType,
1018 StructType, MatrixType, TensorArmType>(
1019 caseFn: [&](auto type) { print(type, os); })
1020 .DefaultUnreachable(message: "Unhandled SPIR-V type");
1021}
1022
1023//===----------------------------------------------------------------------===//
1024// Constant
1025//===----------------------------------------------------------------------===//
1026
1027Operation *SPIRVDialect::materializeConstant(OpBuilder &builder,
1028 Attribute value, Type type,
1029 Location loc) {
1030 if (auto poison = dyn_cast<ub::PoisonAttr>(Val&: value))
1031 return ub::PoisonOp::create(builder, location: loc, result: type, value: poison);
1032
1033 if (!spirv::ConstantOp::isBuildableWith(type))
1034 return nullptr;
1035
1036 return spirv::ConstantOp::create(builder, location: loc, constant: type, value);
1037}
1038
1039//===----------------------------------------------------------------------===//
1040// Shader Interface ABI
1041//===----------------------------------------------------------------------===//
1042
1043LogicalResult SPIRVDialect::verifyOperationAttribute(Operation *op,
1044 NamedAttribute attribute) {
1045 StringRef symbol = attribute.getName().strref();
1046 Attribute attr = attribute.getValue();
1047
1048 if (symbol == spirv::getEntryPointABIAttrName()) {
1049 if (!isa<spirv::EntryPointABIAttr>(Val: attr)) {
1050 return op->emitError(message: "'")
1051 << symbol << "' attribute must be an entry point ABI attribute";
1052 }
1053 } else if (symbol == spirv::getTargetEnvAttrName()) {
1054 if (!isa<spirv::TargetEnvAttr>(Val: attr))
1055 return op->emitError(message: "'") << symbol << "' must be a spirv::TargetEnvAttr";
1056 } else if (symbol == spirv::getLoopControlAttrName()) {
1057 if (!isa<spirv::LoopControlAttr>(Val: attr))
1058 return op->emitError(message: "'")
1059 << symbol << "' must be a spirv::LoopControlAttr";
1060 } else if (symbol == spirv::getSelectionControlAttrName()) {
1061 if (!isa<spirv::SelectionControlAttr>(Val: attr))
1062 return op->emitError(message: "'")
1063 << symbol << "' must be a spirv::SelectionControlAttr";
1064 } else {
1065 return op->emitError(message: "found unsupported '")
1066 << symbol << "' attribute on operation";
1067 }
1068
1069 return success();
1070}
1071
1072/// Verifies the given SPIR-V `attribute` attached to a value of the given
1073/// `valueType` is valid.
1074static LogicalResult verifyRegionAttribute(Location loc, Type valueType,
1075 NamedAttribute attribute) {
1076 StringRef symbol = attribute.getName().strref();
1077 Attribute attr = attribute.getValue();
1078
1079 if (symbol == spirv::getInterfaceVarABIAttrName()) {
1080 auto varABIAttr = dyn_cast<spirv::InterfaceVarABIAttr>(Val&: attr);
1081 if (!varABIAttr)
1082 return emitError(loc, message: "'")
1083 << symbol << "' must be a spirv::InterfaceVarABIAttr";
1084
1085 if (varABIAttr.getStorageClass() && !valueType.isIntOrIndexOrFloat())
1086 return emitError(loc, message: "'") << symbol
1087 << "' attribute cannot specify storage class "
1088 "when attaching to a non-scalar value";
1089 return success();
1090 }
1091 if (symbol == spirv::DecorationAttr::name) {
1092 if (!isa<spirv::DecorationAttr>(Val: attr))
1093 return emitError(loc, message: "'")
1094 << symbol << "' must be a spirv::DecorationAttr";
1095 return success();
1096 }
1097
1098 return emitError(loc, message: "found unsupported '")
1099 << symbol << "' attribute on region argument";
1100}
1101
1102/// Verifies FuncParamAttr decorations. A missing index denotes a function
1103/// result.
1104static LogicalResult verifyFunctionParameterAttributes(
1105 Location loc, Type type, Attribute attr,
1106 std::optional<unsigned> index = std::nullopt) {
1107 auto attrs = dyn_cast<ArrayAttr>(Val&: attr);
1108 if (!attrs || attrs.empty() ||
1109 !llvm::all_of(Range&: attrs,
1110 P: llvm::IsaPred<spirv::FunctionParameterAttributeAttr>))
1111 return emitError(loc, message: "'") << spirv::getFuncParamAttrName()
1112 << "' must be a non-empty array of "
1113 "spirv::FunctionParameterAttributeAttr";
1114
1115 SmallVector<spirv::FunctionParameterAttribute, 8> seen;
1116 bool hasZext = false, hasSext = false;
1117 for (Attribute element : attrs) {
1118 auto value =
1119 cast<spirv::FunctionParameterAttributeAttr>(Val&: element).getValue();
1120 if (llvm::is_contained(Range&: seen, Element: value))
1121 return emitError(loc, message: "duplicate function parameter attribute '")
1122 << spirv::stringifyFunctionParameterAttribute(value) << "'";
1123 seen.push_back(Elt: value);
1124
1125 switch (value) {
1126 case spirv::FunctionParameterAttribute::Zext:
1127 case spirv::FunctionParameterAttribute::Sext:
1128 if (!isa<IntegerType>(Val: type))
1129 return emitError(loc, message: "function parameter attribute '")
1130 << spirv::stringifyFunctionParameterAttribute(value)
1131 << "' requires an integer " << (index ? "parameter" : "result");
1132 hasZext |= value == spirv::FunctionParameterAttribute::Zext;
1133 hasSext |= value == spirv::FunctionParameterAttribute::Sext;
1134 break;
1135 default:
1136 if (!index)
1137 return emitError(loc, message: "function parameter attribute '")
1138 << spirv::stringifyFunctionParameterAttribute(value)
1139 << "' is not valid on function results";
1140 if (!isa<spirv::PointerType>(Val: type))
1141 return emitError(loc, message: "function parameter attribute '")
1142 << spirv::stringifyFunctionParameterAttribute(value)
1143 << "' requires a pointer parameter";
1144 if (value == spirv::FunctionParameterAttribute::Sret && *index != 0)
1145 return emitError(loc,
1146 message: "function parameter attribute 'Sret' requires the "
1147 "first parameter");
1148 break;
1149 }
1150 }
1151 if (hasZext && hasSext)
1152 return emitError(loc, message: "function parameter attributes 'Zext' and 'Sext' are "
1153 "mutually exclusive");
1154 return success();
1155}
1156
1157LogicalResult SPIRVDialect::verifyRegionArgAttribute(Operation *op,
1158 unsigned regionIndex,
1159 unsigned argIndex,
1160 NamedAttribute attribute) {
1161 auto funcOp = dyn_cast<FunctionOpInterface>(Val: op);
1162 if (!funcOp)
1163 return success();
1164 Type argType = funcOp.getArgumentTypes()[argIndex];
1165
1166 if (attribute.getName() == spirv::getFuncParamAlignmentAttrName()) {
1167 auto alignment = dyn_cast<IntegerAttr>(Val: attribute.getValue());
1168 if (!alignment || !alignment.getType().isSignlessInteger(width: 32))
1169 return op->emitError(message: "'")
1170 << spirv::getFuncParamAlignmentAttrName()
1171 << "' must be a 32-bit signless integer attribute";
1172 if (!isa<spirv::PointerType>(Val: argType))
1173 return op->emitError(message: "'") << spirv::getFuncParamAlignmentAttrName()
1174 << "' requires a pointer parameter";
1175 return success();
1176 }
1177 if (attribute.getName() == spirv::getFuncParamAttrName())
1178 return verifyFunctionParameterAttributes(loc: op->getLoc(), type: argType,
1179 attr: attribute.getValue(), index: argIndex);
1180
1181 return verifyRegionAttribute(loc: op->getLoc(), valueType: argType, attribute);
1182}
1183
1184LogicalResult SPIRVDialect::verifyRegionResultAttribute(
1185 Operation *op, unsigned /*regionIndex*/, unsigned resultIndex,
1186 NamedAttribute attribute) {
1187 if (auto funcOp = dyn_cast<spirv::FuncOp>(Val: op)) {
1188 if (attribute.getName() == spirv::getFuncParamAttrName())
1189 return verifyFunctionParameterAttributes(
1190 loc: op->getLoc(), type: funcOp.getResultTypes()[resultIndex],
1191 attr: attribute.getValue());
1192 return op->emitError(message: "found unsupported '")
1193 << attribute.getName().strref() << "' attribute on function result";
1194 }
1195 if (auto graphOp = dyn_cast<spirv::GraphARMOp>(Val: op))
1196 return verifyRegionAttribute(
1197 loc: op->getLoc(), valueType: graphOp.getResultTypes()[resultIndex], attribute);
1198 return op->emitError(
1199 message: "cannot attach SPIR-V attributes to region result which is "
1200 "not part of a spirv::GraphARMOp type");
1201}
1202