1//===- RuntimeVerifiableOpInterfaceImpl.cpp - Op Verification -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Dialect/Tensor/Transforms/RuntimeVerifiableOpInterfaceImpl.h"
10
11#include "mlir/Dialect/Arith/IR/Arith.h"
12#include "mlir/Dialect/Arith/Utils/Utils.h"
13#include "mlir/Dialect/ControlFlow/IR/ControlFlow.h"
14#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
15#include "mlir/Dialect/SCF/IR/SCF.h"
16#include "mlir/Dialect/Tensor/IR/Tensor.h"
17#include "mlir/Interfaces/RuntimeVerifiableOpInterface.h"
18
19using namespace mlir;
20
21namespace mlir {
22namespace tensor {
23namespace {
24/// Generate a runtime check for lb <= value < ub.
25Value generateInBoundsCheck(OpBuilder &builder, Location loc, Value value,
26 Value lb, Value ub) {
27 Value inBounds1 = builder.createOrFold<arith::CmpIOp>(
28 location: loc, args: arith::CmpIPredicate::sge, args&: value, args&: lb);
29 Value inBounds2 = builder.createOrFold<arith::CmpIOp>(
30 location: loc, args: arith::CmpIPredicate::slt, args&: value, args&: ub);
31 Value inBounds =
32 builder.createOrFold<arith::AndIOp>(location: loc, args&: inBounds1, args&: inBounds2);
33 return inBounds;
34}
35
36struct CastOpInterface
37 : public RuntimeVerifiableOpInterface::ExternalModel<CastOpInterface,
38 CastOp> {
39 void
40 generateRuntimeVerification(Operation *op, OpBuilder &builder, Location loc,
41 function_ref<std::string(Operation *, StringRef)>
42 generateErrorMessage) const {
43 auto castOp = cast<CastOp>(Val: op);
44 auto srcType = cast<TensorType>(Val: castOp.getSource().getType());
45
46 // Nothing to check if the result is an unranked tensor.
47 auto resultType = dyn_cast<RankedTensorType>(Val: castOp.getType());
48 if (!resultType)
49 return;
50
51 if (isa<UnrankedTensorType>(Val: srcType)) {
52 // Check rank.
53 Value srcRank = RankOp::create(builder, location: loc, tensor: castOp.getSource());
54 Value resultRank =
55 arith::ConstantIndexOp::create(builder, location: loc, value: resultType.getRank());
56 Value isSameRank = arith::CmpIOp::create(
57 builder, location: loc, predicate: arith::CmpIPredicate::eq, lhs: srcRank, rhs: resultRank);
58 cf::AssertOp::create(builder, location: loc, arg: isSameRank,
59 msg: generateErrorMessage(op, "rank mismatch"));
60 }
61
62 // Check dimension sizes.
63 for (const auto &it : llvm::enumerate(First: resultType.getShape())) {
64 // Static dim size -> static/dynamic dim size does not need verification.
65 if (auto rankedSrcType = dyn_cast<RankedTensorType>(Val&: srcType))
66 if (!rankedSrcType.isDynamicDim(idx: it.index()))
67 continue;
68
69 // Static/dynamic dim size -> dynamic dim size does not need verification.
70 if (resultType.isDynamicDim(idx: it.index()))
71 continue;
72
73 Value srcDimSz =
74 DimOp::create(builder, location: loc, source: castOp.getSource(), index: it.index());
75 Value resultDimSz =
76 arith::ConstantIndexOp::create(builder, location: loc, value: it.value());
77 Value isSameSz = arith::CmpIOp::create(
78 builder, location: loc, predicate: arith::CmpIPredicate::eq, lhs: srcDimSz, rhs: resultDimSz);
79 cf::AssertOp::create(
80 builder, location: loc, arg: isSameSz,
81 msg: generateErrorMessage(op, "size mismatch of dim " +
82 std::to_string(val: it.index())));
83 }
84 }
85};
86
87struct DimOpInterface
88 : public RuntimeVerifiableOpInterface::ExternalModel<DimOpInterface,
89 DimOp> {
90 void
91 generateRuntimeVerification(Operation *op, OpBuilder &builder, Location loc,
92 function_ref<std::string(Operation *, StringRef)>
93 generateErrorMessage) const {
94 auto dimOp = cast<DimOp>(Val: op);
95 Value rank = RankOp::create(builder, location: loc, tensor: dimOp.getSource());
96 Value zero = arith::ConstantIndexOp::create(builder, location: loc, value: 0);
97 cf::AssertOp::create(
98 builder, location: loc,
99 arg: generateInBoundsCheck(builder, loc, value: dimOp.getIndex(), lb: zero, ub: rank),
100 msg: generateErrorMessage(op, "index is out of bounds"));
101 }
102};
103
104/// Verifies that the indices on extract/insert ops are in-bounds of the
105/// tensor's index space: 0 <= index#i < dim#i
106template <typename OpTy>
107struct ExtractInsertOpInterface
108 : public RuntimeVerifiableOpInterface::ExternalModel<
109 ExtractInsertOpInterface<OpTy>, OpTy> {
110 void
111 generateRuntimeVerification(Operation *op, OpBuilder &builder, Location loc,
112 function_ref<std::string(Operation *, StringRef)>
113 generateErrorMessage) const {
114 auto extractInsertOp = cast<OpTy>(op);
115
116 Value tensor;
117 if constexpr (std::is_same_v<OpTy, ExtractOp>) {
118 tensor = extractInsertOp.getTensor();
119 } else if constexpr (std::is_same_v<OpTy, InsertOp>) {
120 tensor = extractInsertOp.getDest();
121 } else {
122 llvm_unreachable("invalid op");
123 }
124 auto tensorType = cast<RankedTensorType>(Val: tensor.getType());
125 auto rank = tensorType.getRank();
126 if (rank == 0) {
127 // Nothing to check for 0-d tensors.
128 return;
129 }
130
131 auto indices = extractInsertOp.getIndices();
132 auto zero = arith::ConstantIndexOp::create(builder, location: loc, value: 0);
133 Value assertCond;
134 for (auto i : llvm::seq<int64_t>(Begin: 0, End: rank)) {
135 Value dimOp = builder.createOrFold<tensor::DimOp>(location: loc, args&: tensor, args&: i);
136 Value inBounds =
137 generateInBoundsCheck(builder, loc, indices[i], zero, dimOp);
138 assertCond =
139 i > 0 ? builder.createOrFold<arith::AndIOp>(location: loc, args&: assertCond, args&: inBounds)
140 : inBounds;
141 }
142 cf::AssertOp::create(builder, location: loc, arg: assertCond,
143 msg: generateErrorMessage(op, "out-of-bounds access"));
144 }
145};
146
147struct ExtractSliceOpInterface
148 : public RuntimeVerifiableOpInterface::ExternalModel<
149 ExtractSliceOpInterface, ExtractSliceOp> {
150 void
151 generateRuntimeVerification(Operation *op, OpBuilder &builder, Location loc,
152 function_ref<std::string(Operation *, StringRef)>
153 generateErrorMessage) const {
154 auto extractSliceOp = cast<ExtractSliceOp>(Val: op);
155 RankedTensorType sourceType = extractSliceOp.getSource().getType();
156
157 // For each dimension, assert that:
158 // For empty slices (size == 0) : 0 <= offset <= dim_size
159 // For non-empty slices (size > 0): 0 <= offset < dim_size
160 // 0 <= offset + (size - 1) * stride <
161 // dim_size
162 Value zero = arith::ConstantIndexOp::create(builder, location: loc, value: 0);
163 Value one = arith::ConstantIndexOp::create(builder, location: loc, value: 1);
164
165 for (int64_t i : llvm::seq<int64_t>(Begin: 0, End: sourceType.getRank())) {
166
167 builder.setInsertionPoint(extractSliceOp);
168
169 Value offset = getValueOrCreateConstantIndexOp(
170 b&: builder, loc, ofr: extractSliceOp.getMixedOffsets()[i]);
171 Value size = getValueOrCreateConstantIndexOp(
172 b&: builder, loc, ofr: extractSliceOp.getMixedSizes()[i]);
173 Value stride = getValueOrCreateConstantIndexOp(
174 b&: builder, loc, ofr: extractSliceOp.getMixedStrides()[i]);
175 Value dimSize = builder.createOrFold<tensor::DimOp>(
176 location: loc, args: extractSliceOp.getSource(), args&: i);
177
178 // Verify that offset is in-bounds (conditional on slice size).
179 Value sizeIsZero = arith::CmpIOp::create(
180 builder, location: loc, predicate: arith::CmpIPredicate::eq, lhs: size, rhs: zero);
181 auto offsetCheckIf = scf::IfOp::create(
182 builder, location: loc, cond: sizeIsZero,
183 thenBuilder: [&](OpBuilder &b, Location loc) {
184 // For empty slices, offset can be at the boundary: 0 <= offset <=
185 // dimSize.
186 Value offsetGEZero = arith::CmpIOp::create(
187 builder&: b, location: loc, predicate: arith::CmpIPredicate::sge, lhs: offset, rhs: zero);
188 Value offsetLEDimSize = arith::CmpIOp::create(
189 builder&: b, location: loc, predicate: arith::CmpIPredicate::sle, lhs: offset, rhs: dimSize);
190 Value emptyOffsetValid =
191 arith::AndIOp::create(builder&: b, location: loc, lhs: offsetGEZero, rhs: offsetLEDimSize);
192 scf::YieldOp::create(builder&: b, location: loc, results: emptyOffsetValid);
193 },
194 elseBuilder: [&](OpBuilder &b, Location loc) {
195 // For non-empty slices, offset must be a valid index: 0 <= offset <
196 // dimSize.
197 Value offsetInBounds =
198 generateInBoundsCheck(builder&: b, loc, value: offset, lb: zero, ub: dimSize);
199 scf::YieldOp::create(builder&: b, location: loc, results: offsetInBounds);
200 });
201
202 Value offsetCondition = offsetCheckIf.getResult(i: 0);
203 cf::AssertOp::create(builder, location: loc, arg: offsetCondition,
204 msg: generateErrorMessage(op, "offset " +
205 std::to_string(val: i) +
206 " is out-of-bounds"));
207
208 // Verify that the slice endpoint is in-bounds (only for non-empty
209 // slices).
210 Value sizeIsNonZero = arith::CmpIOp::create(
211 builder, location: loc, predicate: arith::CmpIPredicate::sgt, lhs: size, rhs: zero);
212 auto ifOp = scf::IfOp::create(
213 builder, location: loc, cond: sizeIsNonZero,
214 thenBuilder: [&](OpBuilder &b, Location loc) {
215 // Verify that slice does not run out-of-bounds.
216 Value sizeMinusOne = arith::SubIOp::create(builder&: b, location: loc, lhs: size, rhs: one);
217 Value sizeMinusOneTimesStride =
218 arith::MulIOp::create(builder&: b, location: loc, lhs: sizeMinusOne, rhs: stride);
219 Value lastPos =
220 arith::AddIOp::create(builder&: b, location: loc, lhs: offset, rhs: sizeMinusOneTimesStride);
221 Value lastPosInBounds =
222 generateInBoundsCheck(builder&: b, loc, value: lastPos, lb: zero, ub: dimSize);
223 scf::YieldOp::create(builder&: b, location: loc, results: lastPosInBounds);
224 },
225 elseBuilder: [&](OpBuilder &b, Location loc) {
226 Value trueVal =
227 arith::ConstantOp::create(builder&: b, location: loc, value: b.getBoolAttr(value: true));
228 scf::YieldOp::create(builder&: b, location: loc, results: trueVal);
229 });
230
231 Value finalCondition = ifOp.getResult(i: 0);
232 cf::AssertOp::create(
233 builder, location: loc, arg: finalCondition,
234 msg: generateErrorMessage(
235 op, "extract_slice runs out-of-bounds along dimension " +
236 std::to_string(val: i)));
237 }
238 }
239};
240} // namespace
241} // namespace tensor
242} // namespace mlir
243
244void mlir::tensor::registerRuntimeVerifiableOpInterfaceExternalModels(
245 DialectRegistry &registry) {
246 registry.addExtension(extensionFn: +[](MLIRContext *ctx, tensor::TensorDialect *dialect) {
247 CastOp::attachInterface<CastOpInterface>(context&: *ctx);
248 DimOp::attachInterface<DimOpInterface>(context&: *ctx);
249 ExtractOp::attachInterface<ExtractInsertOpInterface<ExtractOp>>(context&: *ctx);
250 ExtractSliceOp::attachInterface<ExtractSliceOpInterface>(context&: *ctx);
251 InsertOp::attachInterface<ExtractInsertOpInterface<InsertOp>>(context&: *ctx);
252
253 // Load additional dialects of which ops may get created.
254 ctx->loadDialect<arith::ArithDialect, cf::ControlFlowDialect>();
255 });
256}
257