| 1 | //===--- TosaProfileCompliance.cpp - Tosa Profile Compliance Validation ---===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Dialect/Tosa/IR/TosaProfileCompliance.h" |
| 10 | #include "llvm/ADT/StringExtras.h" |
| 11 | #include "llvm/Support/Compiler.h" |
| 12 | #include "llvm/Support/raw_ostream.h" |
| 13 | |
| 14 | namespace mlir { |
| 15 | namespace tosa { |
| 16 | LLVM_ATTRIBUTE_NOINLINE |
| 17 | VersionedTypeInfo::VersionedTypeInfo(std::initializer_list<TypeInfoRef> types, |
| 18 | SpecificationVersion version) |
| 19 | : second(version) { |
| 20 | first.reserve(N: types.size()); |
| 21 | for (const TypeInfoRef &type : types) |
| 22 | first.push_back(Elt: *type.ptr); |
| 23 | } |
| 24 | |
| 25 | LLVM_ATTRIBUTE_NOINLINE |
| 26 | VersionedTypeInfo::VersionedTypeInfo(SmallVector<TypeInfo> types, |
| 27 | SpecificationVersion version) |
| 28 | : first(std::move(types)), second(version) {} |
| 29 | |
| 30 | LLVM_ATTRIBUTE_NOINLINE |
| 31 | VersionedTypeInfo::VersionedTypeInfo(const VersionedTypeInfo &other) = default; |
| 32 | |
| 33 | LLVM_ATTRIBUTE_NOINLINE |
| 34 | VersionedTypeInfo::VersionedTypeInfo(VersionedTypeInfo &&other) = default; |
| 35 | |
| 36 | LLVM_ATTRIBUTE_NOINLINE |
| 37 | VersionedTypeInfo & |
| 38 | VersionedTypeInfo::operator=(const VersionedTypeInfo &other) = default; |
| 39 | |
| 40 | LLVM_ATTRIBUTE_NOINLINE |
| 41 | VersionedTypeInfo & |
| 42 | VersionedTypeInfo::operator=(VersionedTypeInfo &&other) = default; |
| 43 | |
| 44 | LLVM_ATTRIBUTE_NOINLINE |
| 45 | VersionedTypeInfo::~VersionedTypeInfo() = default; |
| 46 | |
| 47 | template <typename T> |
| 48 | LLVM_ATTRIBUTE_NOINLINE OpComplianceInfo<T>::OpComplianceInfo( |
| 49 | std::initializer_list<T> mode, |
| 50 | std::initializer_list<VersionedTypeInfo> operandTypeInfoSet, |
| 51 | CheckCondition condition) |
| 52 | : mode(mode), operandTypeInfoSet(operandTypeInfoSet), condition(condition) { |
| 53 | } |
| 54 | |
| 55 | template <typename T> |
| 56 | LLVM_ATTRIBUTE_NOINLINE OpComplianceInfo<T>::OpComplianceInfo( |
| 57 | SmallVector<T> mode, SmallVector<VersionedTypeInfo> operandTypeInfoSet, |
| 58 | CheckCondition condition) |
| 59 | : mode(std::move(mode)), operandTypeInfoSet(std::move(operandTypeInfoSet)), |
| 60 | condition(condition) {} |
| 61 | |
| 62 | template <typename T> |
| 63 | LLVM_ATTRIBUTE_NOINLINE |
| 64 | OpComplianceInfo<T>::OpComplianceInfo(const OpComplianceInfo &other) = default; |
| 65 | |
| 66 | template <typename T> |
| 67 | LLVM_ATTRIBUTE_NOINLINE |
| 68 | OpComplianceInfo<T>::OpComplianceInfo(OpComplianceInfo &&other) = default; |
| 69 | |
| 70 | template <typename T> |
| 71 | LLVM_ATTRIBUTE_NOINLINE OpComplianceInfo<T> & |
| 72 | OpComplianceInfo<T>::operator=(const OpComplianceInfo &other) = default; |
| 73 | |
| 74 | template <typename T> |
| 75 | LLVM_ATTRIBUTE_NOINLINE OpComplianceInfo<T> & |
| 76 | OpComplianceInfo<T>::operator=(OpComplianceInfo &&other) = default; |
| 77 | |
| 78 | template <typename T> |
| 79 | LLVM_ATTRIBUTE_NOINLINE OpComplianceInfo<T>::~OpComplianceInfo() = default; |
| 80 | |
| 81 | template struct OpComplianceInfo<Profile>; |
| 82 | template struct OpComplianceInfo<Extension>; |
| 83 | |
| 84 | TosaProfileCompliance::TosaProfileCompliance() { |
| 85 | static const TypeInfo boolT = {mlir::IntegerType::getTypeID(), 1}; |
| 86 | static const TypeInfo i4T = {mlir::IntegerType::getTypeID(), 4}; |
| 87 | static const TypeInfo i8T = {mlir::IntegerType::getTypeID(), 8}; |
| 88 | static const TypeInfo i16T = {mlir::IntegerType::getTypeID(), 16}; |
| 89 | static const TypeInfo i32T = {mlir::IntegerType::getTypeID(), 32}; |
| 90 | static const TypeInfo i48T = {mlir::IntegerType::getTypeID(), 48}; |
| 91 | static const TypeInfo i64T = {mlir::IntegerType::getTypeID(), 64}; |
| 92 | static const TypeInfo bf16T = {mlir::BFloat16Type::getTypeID(), 16}; |
| 93 | static const TypeInfo fp16T = {mlir::Float16Type::getTypeID(), 16}; |
| 94 | static const TypeInfo fp32T = {mlir::Float32Type::getTypeID(), 32}; |
| 95 | static const TypeInfo fp8e4m3T = {mlir::Float8E4M3FNType::getTypeID(), 8}; |
| 96 | static const TypeInfo fp8e5m2T = {mlir::Float8E5M2Type::getTypeID(), 8}; |
| 97 | |
| 98 | // micro-scaling formats |
| 99 | // Note: these types exist to support the deprecated block_scaled operations |
| 100 | // and can be removed once those operations are removed. |
| 101 | static const TypeInfo fp6e2m3T = {mlir::Float6E2M3FNType::getTypeID(), 6}; |
| 102 | static const TypeInfo fp6e3m2T = {mlir::Float6E3M2FNType::getTypeID(), 6}; |
| 103 | static const TypeInfo fp4e2m1T = {mlir::Float4E2M1FNType::getTypeID(), 4}; |
| 104 | static const TypeInfo fp8ue8m0T = {mlir::Float8E8M0FNUType::getTypeID(), 8}; |
| 105 | static const TypeInfo mxint8T = {mlir::MXI8Type::getTypeID(), 8}; |
| 106 | |
| 107 | // Block scaled formats |
| 108 | const TypeID blockScaledID = mlir::tosa::BlockScaledType::getTypeID(); |
| 109 | const TypeID fp4e2m1ID = mlir::Float4E2M1FNType::getTypeID(); |
| 110 | const TypeID fp6e2m3ID = mlir::Float6E2M3FNType::getTypeID(); |
| 111 | const TypeID fp6e3m2ID = mlir::Float6E3M2FNType::getTypeID(); |
| 112 | const TypeID fp8e4m3ID = mlir::Float8E4M3FNType::getTypeID(); |
| 113 | const TypeID fp8e5m2ID = mlir::Float8E5M2Type::getTypeID(); |
| 114 | const TypeID fp8ue8m0ID = mlir::Float8E8M0FNUType::getTypeID(); |
| 115 | const TypeID mxint8ID = mlir::MXI8Type::getTypeID(); |
| 116 | |
| 117 | static const TypeInfo bs32_fp8ue8m0_fp4e2m1T = { |
| 118 | blockScaledID, 4, fp4e2m1ID, fp8ue8m0ID, |
| 119 | tosa::BlockShape::BLOCK_SHAPE_32}; |
| 120 | static const TypeInfo bs32_fp8ue8m0_fp6e2m3T = { |
| 121 | blockScaledID, 6, fp6e2m3ID, fp8ue8m0ID, |
| 122 | tosa::BlockShape::BLOCK_SHAPE_32}; |
| 123 | static const TypeInfo bs32_fp8ue8m0_fp6e3m2T = { |
| 124 | blockScaledID, 6, fp6e3m2ID, fp8ue8m0ID, |
| 125 | tosa::BlockShape::BLOCK_SHAPE_32}; |
| 126 | static const TypeInfo bs32_fp8ue8m0_fp8e4m3T = { |
| 127 | blockScaledID, 8, fp8e4m3ID, fp8ue8m0ID, |
| 128 | tosa::BlockShape::BLOCK_SHAPE_32}; |
| 129 | static const TypeInfo bs32_fp8ue8m0_fp8e5m2T = { |
| 130 | blockScaledID, 8, fp8e5m2ID, fp8ue8m0ID, |
| 131 | tosa::BlockShape::BLOCK_SHAPE_32}; |
| 132 | static const TypeInfo bs32_fp8ue8m0_mxint8T = { |
| 133 | blockScaledID, 8, mxint8ID, fp8ue8m0ID, tosa::BlockShape::BLOCK_SHAPE_32}; |
| 134 | |
| 135 | // The profile-based compliance content below is auto-generated by a script |
| 136 | // in https://github.com/arm/tosa-specification |
| 137 | #include "mlir/Dialect/Tosa/IR/TosaComplianceData.h.inc" |
| 138 | // End of auto-generated metadata |
| 139 | } |
| 140 | |
| 141 | template <> |
| 142 | const OperationProfileComplianceMap & |
| 143 | TosaProfileCompliance::getProfileComplianceMap<Profile>() const { |
| 144 | return profileComplianceMap; |
| 145 | } |
| 146 | |
| 147 | template <> |
| 148 | const OperationExtensionComplianceMap & |
| 149 | TosaProfileCompliance::getProfileComplianceMap<Extension>() const { |
| 150 | return extensionComplianceMap; |
| 151 | } |
| 152 | |
| 153 | // Base populating function |
| 154 | LogicalResult ProfileInfoDepot::populateProfileInfo(ValueRange operands, |
| 155 | ValueRange results) { |
| 156 | for (const auto &operand : operands) |
| 157 | addValue(v: operand); |
| 158 | for (const auto &result : results) |
| 159 | addValue(v: result); |
| 160 | return success(); |
| 161 | } |
| 162 | |
| 163 | template <> |
| 164 | LogicalResult ProfileInfoDepot::populateProfileInfo(tosa::ConcatOp op) { |
| 165 | addValue(v: op.getInput1().front()); |
| 166 | addValue(v: op.getOutput()); |
| 167 | return success(); |
| 168 | } |
| 169 | |
| 170 | template <> |
| 171 | LogicalResult ProfileInfoDepot::populateProfileInfo(tosa::AvgPool2dOp op) { |
| 172 | addValue(v: op.getInput()); |
| 173 | addValue(v: op.getInputZp()); |
| 174 | addValue(v: op.getOutputZp()); |
| 175 | addType(t: op.getAccType()); |
| 176 | addValue(v: op.getOutput()); |
| 177 | return success(); |
| 178 | } |
| 179 | |
| 180 | template <> |
| 181 | LogicalResult |
| 182 | ProfileInfoDepot::populateProfileInfo(tosa::AvgPool2dAdaptiveOp op) { |
| 183 | addValue(v: op.getInput()); |
| 184 | addValue(v: op.getInputZp()); |
| 185 | addValue(v: op.getOutputZp()); |
| 186 | addType(t: op.getAccType()); |
| 187 | addValue(v: op.getOutput()); |
| 188 | return success(); |
| 189 | } |
| 190 | |
| 191 | template <> |
| 192 | LogicalResult |
| 193 | ProfileInfoDepot::populateProfileInfo(tosa::MaxPool2dAdaptiveOp op) { |
| 194 | addValue(v: op.getInput()); |
| 195 | addValue(v: op.getOutput()); |
| 196 | return success(); |
| 197 | } |
| 198 | |
| 199 | template <typename T> |
| 200 | LogicalResult ProfileInfoDepot::populateProfileInfoConv(T op) { |
| 201 | addValue(v: op.getInput()); |
| 202 | addValue(v: op.getWeight()); |
| 203 | addValue(v: op.getBias()); |
| 204 | addValue(v: op.getInputZp()); |
| 205 | addValue(v: op.getWeightZp()); |
| 206 | addType(t: op.getAccType()); |
| 207 | addValue(v: op.getOutput()); |
| 208 | return success(); |
| 209 | } |
| 210 | |
| 211 | template <> |
| 212 | LogicalResult ProfileInfoDepot::populateProfileInfo(tosa::Conv2DOp op) { |
| 213 | return populateProfileInfoConv(op); |
| 214 | } |
| 215 | |
| 216 | template <> |
| 217 | LogicalResult ProfileInfoDepot::populateProfileInfo(tosa::Conv3DOp op) { |
| 218 | return populateProfileInfoConv(op); |
| 219 | } |
| 220 | |
| 221 | template <> |
| 222 | LogicalResult |
| 223 | ProfileInfoDepot::populateProfileInfo(tosa::TransposeConv2DOp op) { |
| 224 | return populateProfileInfoConv(op); |
| 225 | } |
| 226 | |
| 227 | template <> |
| 228 | LogicalResult |
| 229 | ProfileInfoDepot::populateProfileInfo(tosa::DepthwiseConv2DOp op) { |
| 230 | return populateProfileInfoConv(op); |
| 231 | } |
| 232 | |
| 233 | template <> |
| 234 | LogicalResult |
| 235 | ProfileInfoDepot::populateProfileInfo(tosa::Conv2DBlockScaledOp op) { |
| 236 | addValue(v: op.getInputData()); |
| 237 | addValue(v: op.getInputScale()); |
| 238 | addValue(v: op.getWeightData()); |
| 239 | addValue(v: op.getWeightScale()); |
| 240 | addValue(v: op.getBias()); |
| 241 | addValue(v: op.getOutput()); |
| 242 | return success(); |
| 243 | } |
| 244 | |
| 245 | template <> |
| 246 | LogicalResult ProfileInfoDepot::populateProfileInfo(tosa::PadOp op) { |
| 247 | addValue(v: op.getInput1()); |
| 248 | addValue(v: op.getPadConst()); |
| 249 | addValue(v: op.getOutput()); |
| 250 | return success(); |
| 251 | } |
| 252 | |
| 253 | template <typename T> |
| 254 | LogicalResult ProfileInfoDepot::populateProfileInfoDataLayout(T op) { |
| 255 | addValue(v: op.getInput1()); |
| 256 | addValue(v: op.getOutput()); |
| 257 | return success(); |
| 258 | } |
| 259 | |
| 260 | template <> |
| 261 | LogicalResult ProfileInfoDepot::populateProfileInfo(tosa::ReshapeOp op) { |
| 262 | return populateProfileInfoDataLayout(op); |
| 263 | } |
| 264 | |
| 265 | template <> |
| 266 | LogicalResult ProfileInfoDepot::populateProfileInfo(tosa::SliceOp op) { |
| 267 | return populateProfileInfoDataLayout(op); |
| 268 | } |
| 269 | |
| 270 | template <> |
| 271 | LogicalResult ProfileInfoDepot::populateProfileInfo(tosa::TileOp op) { |
| 272 | return populateProfileInfoDataLayout(op); |
| 273 | } |
| 274 | |
| 275 | template <> |
| 276 | LogicalResult ProfileInfoDepot::populateProfileInfo(tosa::TransposeOp op) { |
| 277 | return populateProfileInfoDataLayout(op); |
| 278 | } |
| 279 | |
| 280 | template <> |
| 281 | LogicalResult ProfileInfoDepot::populateProfileInfo(tosa::GatherOp op) { |
| 282 | addValue(v: op.getValues()); |
| 283 | addValue(v: op.getIndices()); |
| 284 | addValue(v: op.getOutput()); |
| 285 | return success(); |
| 286 | } |
| 287 | |
| 288 | template <> |
| 289 | LogicalResult ProfileInfoDepot::populateProfileInfo(tosa::RowGatherOp op) { |
| 290 | addValue(v: op.getValues()); |
| 291 | addValue(v: op.getIndices()); |
| 292 | addValue(v: op.getOutput()); |
| 293 | return success(); |
| 294 | } |
| 295 | |
| 296 | template <> |
| 297 | LogicalResult |
| 298 | ProfileInfoDepot::populateProfileInfo(tosa::RowGatherBlockScaledOp op) { |
| 299 | for (Value value : op.getValues()) |
| 300 | addValue(v: value); |
| 301 | addValue(v: op.getIndices()); |
| 302 | addValue(v: op.getRowCount()); |
| 303 | for (Value result : op.getOutput()) |
| 304 | addValue(v: result); |
| 305 | return success(); |
| 306 | } |
| 307 | |
| 308 | template <> |
| 309 | LogicalResult ProfileInfoDepot::populateProfileInfo(tosa::ScatterOp op) { |
| 310 | addValue(v: op.getValuesIn()); |
| 311 | addValue(v: op.getIndices()); |
| 312 | addValue(v: op.getInput()); |
| 313 | addValue(v: op.getValuesOut()); |
| 314 | return success(); |
| 315 | } |
| 316 | |
| 317 | template <> |
| 318 | LogicalResult ProfileInfoDepot::populateProfileInfo(tosa::MulOp op) { |
| 319 | addValue(v: op.getInput1()); |
| 320 | addValue(v: op.getInput2()); |
| 321 | addValue(v: op.getOutput()); |
| 322 | return success(); |
| 323 | } |
| 324 | |
| 325 | template <> |
| 326 | LogicalResult ProfileInfoDepot::populateProfileInfo(tosa::ResizeOp op) { |
| 327 | addValue(v: op.getInput()); |
| 328 | addValue(v: op.getOutput()); |
| 329 | return success(); |
| 330 | } |
| 331 | |
| 332 | template <> |
| 333 | LogicalResult ProfileInfoDepot::populateProfileInfo(tosa::SelectOp op) { |
| 334 | addValue(v: op.getOnTrue()); |
| 335 | addValue(v: op.getOnFalse()); |
| 336 | addValue(v: op.getOutput()); |
| 337 | return success(); |
| 338 | } |
| 339 | |
| 340 | template <> |
| 341 | LogicalResult ProfileInfoDepot::populateProfileInfo(tosa::RescaleOp op) { |
| 342 | addValue(v: op.getInput()); |
| 343 | addValue(v: op.getInputZp()); |
| 344 | addValue(v: op.getOutputZp()); |
| 345 | addValue(v: op.getOutput()); |
| 346 | return success(); |
| 347 | } |
| 348 | |
| 349 | template <> |
| 350 | LogicalResult ProfileInfoDepot::populateProfileInfo(tosa::MatMulOp op) { |
| 351 | addValue(v: op.getA()); |
| 352 | addValue(v: op.getB()); |
| 353 | addValue(v: op.getAZp()); |
| 354 | addValue(v: op.getBZp()); |
| 355 | addValue(v: op.getOutput()); |
| 356 | return success(); |
| 357 | } |
| 358 | |
| 359 | template <> |
| 360 | LogicalResult ProfileInfoDepot::populateProfileInfo(tosa::MatMulTOp op) { |
| 361 | addValue(v: op.getA()); |
| 362 | addValue(v: op.getB()); |
| 363 | addValue(v: op.getAZp()); |
| 364 | addValue(v: op.getBZp()); |
| 365 | addValue(v: op.getOutput()); |
| 366 | return success(); |
| 367 | } |
| 368 | |
| 369 | template <> |
| 370 | LogicalResult ProfileInfoDepot::populateProfileInfo(tosa::VariableOp op) { |
| 371 | addType(t: op.getType()); |
| 372 | return success(); |
| 373 | } |
| 374 | |
| 375 | template <> |
| 376 | LogicalResult ProfileInfoDepot::populateProfileInfo(tosa::VariableWriteOp op) { |
| 377 | addValue(v: op.getInput1()); |
| 378 | return success(); |
| 379 | } |
| 380 | |
| 381 | template <> |
| 382 | LogicalResult ProfileInfoDepot::populateProfileInfo(tosa::DimOp op) { |
| 383 | addValue(v: op.getInput1()); |
| 384 | return success(); |
| 385 | } |
| 386 | |
| 387 | LogicalResult ProfileInfoDepot::populatationDispatch(Operation *op) { |
| 388 | // This helper function only populates the info for the customised operands. |
| 389 | #define POPULATE_PROFILE_INFO_CUSTOM(tosaOp) \ |
| 390 | if (isa<tosa::tosaOp##Op>(op)) { \ |
| 391 | return populateProfileInfo(cast<tosa::tosaOp##Op>(op)); \ |
| 392 | } |
| 393 | |
| 394 | #define POPULATE_PROFILE_INFO_SKIP(tosaOp) \ |
| 395 | if (isa<tosa::tosaOp##Op>(op)) \ |
| 396 | return success(); |
| 397 | |
| 398 | // This helper function populates the info for all operands. |
| 399 | #define POPULATE_PROFILE_INFO_COMMON(tosaOp) \ |
| 400 | if (isa<tosa::tosaOp##Op>(op)) { \ |
| 401 | return populateProfileInfo(op->getOperands(), op->getResults()); \ |
| 402 | } |
| 403 | |
| 404 | // Skip irrelevant operands when they are independent and not tied to any |
| 405 | // specific profile/extension. |
| 406 | POPULATE_PROFILE_INFO_CUSTOM(AvgPool2d) |
| 407 | POPULATE_PROFILE_INFO_CUSTOM(AvgPool2dAdaptive) |
| 408 | POPULATE_PROFILE_INFO_CUSTOM(TransposeConv2D) |
| 409 | POPULATE_PROFILE_INFO_CUSTOM(Conv2D) |
| 410 | POPULATE_PROFILE_INFO_CUSTOM(Conv2DBlockScaled) |
| 411 | POPULATE_PROFILE_INFO_CUSTOM(Conv3D) |
| 412 | POPULATE_PROFILE_INFO_CUSTOM(DepthwiseConv2D) |
| 413 | POPULATE_PROFILE_INFO_CUSTOM(Mul) |
| 414 | POPULATE_PROFILE_INFO_CUSTOM(Concat) |
| 415 | POPULATE_PROFILE_INFO_CUSTOM(Pad) |
| 416 | POPULATE_PROFILE_INFO_CUSTOM(Reshape) |
| 417 | POPULATE_PROFILE_INFO_CUSTOM(Slice) |
| 418 | POPULATE_PROFILE_INFO_CUSTOM(Tile) |
| 419 | POPULATE_PROFILE_INFO_CUSTOM(Transpose) |
| 420 | POPULATE_PROFILE_INFO_CUSTOM(Gather) |
| 421 | POPULATE_PROFILE_INFO_CUSTOM(RowGather) |
| 422 | POPULATE_PROFILE_INFO_CUSTOM(RowGatherBlockScaled) |
| 423 | POPULATE_PROFILE_INFO_CUSTOM(Scatter) |
| 424 | POPULATE_PROFILE_INFO_CUSTOM(Resize) |
| 425 | POPULATE_PROFILE_INFO_CUSTOM(Select) |
| 426 | POPULATE_PROFILE_INFO_CUSTOM(Rescale) |
| 427 | POPULATE_PROFILE_INFO_CUSTOM(MatMul) |
| 428 | POPULATE_PROFILE_INFO_CUSTOM(MatMulT) |
| 429 | POPULATE_PROFILE_INFO_CUSTOM(Variable) |
| 430 | POPULATE_PROFILE_INFO_CUSTOM(VariableWrite) |
| 431 | POPULATE_PROFILE_INFO_CUSTOM(Dim) |
| 432 | POPULATE_PROFILE_INFO_CUSTOM(MaxPool2dAdaptive) |
| 433 | |
| 434 | // For the most of tosa operators, all operands are profile/extension related |
| 435 | // and hence are all considered in this profile-based compilance check. |
| 436 | POPULATE_PROFILE_INFO_COMMON(MatmulTBlockScaled) |
| 437 | POPULATE_PROFILE_INFO_COMMON(FFT2d) |
| 438 | POPULATE_PROFILE_INFO_COMMON(RFFT2d) |
| 439 | POPULATE_PROFILE_INFO_COMMON(Cast) |
| 440 | POPULATE_PROFILE_INFO_COMMON(CastFromBlockScaled) |
| 441 | POPULATE_PROFILE_INFO_COMMON(CastToBlockScaled) |
| 442 | POPULATE_PROFILE_INFO_COMMON(Const) |
| 443 | POPULATE_PROFILE_INFO_COMMON(ArgMax) |
| 444 | POPULATE_PROFILE_INFO_COMMON(ArgMin) |
| 445 | POPULATE_PROFILE_INFO_COMMON(Sub) |
| 446 | POPULATE_PROFILE_INFO_COMMON(Maximum) |
| 447 | POPULATE_PROFILE_INFO_COMMON(Minimum) |
| 448 | POPULATE_PROFILE_INFO_COMMON(MaxPool2d) |
| 449 | POPULATE_PROFILE_INFO_COMMON(Clamp) |
| 450 | POPULATE_PROFILE_INFO_COMMON(Erf) |
| 451 | POPULATE_PROFILE_INFO_COMMON(Sigmoid) |
| 452 | POPULATE_PROFILE_INFO_COMMON(Tanh) |
| 453 | POPULATE_PROFILE_INFO_COMMON(Add) |
| 454 | POPULATE_PROFILE_INFO_COMMON(ArithmeticRightShift) |
| 455 | POPULATE_PROFILE_INFO_COMMON(BitwiseAnd) |
| 456 | POPULATE_PROFILE_INFO_COMMON(BitwiseNot) |
| 457 | POPULATE_PROFILE_INFO_COMMON(BitwiseOr) |
| 458 | POPULATE_PROFILE_INFO_COMMON(BitwiseXor) |
| 459 | POPULATE_PROFILE_INFO_COMMON(LogicalLeftShift) |
| 460 | POPULATE_PROFILE_INFO_COMMON(LogicalRightShift) |
| 461 | POPULATE_PROFILE_INFO_COMMON(LogicalAnd) |
| 462 | POPULATE_PROFILE_INFO_COMMON(LogicalNot) |
| 463 | POPULATE_PROFILE_INFO_COMMON(LogicalOr) |
| 464 | POPULATE_PROFILE_INFO_COMMON(LogicalXor) |
| 465 | POPULATE_PROFILE_INFO_COMMON(IntDiv) |
| 466 | POPULATE_PROFILE_INFO_COMMON(Pow) |
| 467 | POPULATE_PROFILE_INFO_COMMON(Table) |
| 468 | POPULATE_PROFILE_INFO_COMMON(Abs) |
| 469 | POPULATE_PROFILE_INFO_COMMON(Ceil) |
| 470 | POPULATE_PROFILE_INFO_COMMON(Clz) |
| 471 | POPULATE_PROFILE_INFO_COMMON(Sin) |
| 472 | POPULATE_PROFILE_INFO_COMMON(Cos) |
| 473 | POPULATE_PROFILE_INFO_COMMON(Exp) |
| 474 | POPULATE_PROFILE_INFO_COMMON(Floor) |
| 475 | POPULATE_PROFILE_INFO_COMMON(Log) |
| 476 | POPULATE_PROFILE_INFO_COMMON(Negate) |
| 477 | POPULATE_PROFILE_INFO_COMMON(Reciprocal) |
| 478 | POPULATE_PROFILE_INFO_COMMON(Rsqrt) |
| 479 | POPULATE_PROFILE_INFO_COMMON(ReduceAll) |
| 480 | POPULATE_PROFILE_INFO_COMMON(ReduceAny) |
| 481 | POPULATE_PROFILE_INFO_COMMON(ReduceMax) |
| 482 | POPULATE_PROFILE_INFO_COMMON(ReduceMin) |
| 483 | POPULATE_PROFILE_INFO_COMMON(ReduceProduct) |
| 484 | POPULATE_PROFILE_INFO_COMMON(ReduceSum) |
| 485 | POPULATE_PROFILE_INFO_COMMON(Equal) |
| 486 | POPULATE_PROFILE_INFO_COMMON(GreaterEqual) |
| 487 | POPULATE_PROFILE_INFO_COMMON(Greater) |
| 488 | POPULATE_PROFILE_INFO_COMMON(Reverse) |
| 489 | POPULATE_PROFILE_INFO_COMMON(Identity) |
| 490 | POPULATE_PROFILE_INFO_COMMON(VariableRead) |
| 491 | |
| 492 | // Type Invariant Extension, a capability extension that is independent |
| 493 | // of the data type, meaning any compatible type can be used. No type |
| 494 | // constraint for those operations. |
| 495 | POPULATE_PROFILE_INFO_SKIP(AddShape) |
| 496 | POPULATE_PROFILE_INFO_SKIP(AssertEqualShape) |
| 497 | POPULATE_PROFILE_INFO_SKIP(ConcatShape) |
| 498 | POPULATE_PROFILE_INFO_SKIP(ConstShape) |
| 499 | POPULATE_PROFILE_INFO_SKIP(DivCeilShape) |
| 500 | POPULATE_PROFILE_INFO_SKIP(DivFloorShape) |
| 501 | POPULATE_PROFILE_INFO_SKIP(Exp2Shape) |
| 502 | POPULATE_PROFILE_INFO_SKIP(Log2CeilShape) |
| 503 | POPULATE_PROFILE_INFO_SKIP(Log2FloorShape) |
| 504 | POPULATE_PROFILE_INFO_SKIP(MaxShape) |
| 505 | POPULATE_PROFILE_INFO_SKIP(MinShape) |
| 506 | POPULATE_PROFILE_INFO_SKIP(ModShape) |
| 507 | POPULATE_PROFILE_INFO_SKIP(MulShape) |
| 508 | POPULATE_PROFILE_INFO_SKIP(SliceShape) |
| 509 | POPULATE_PROFILE_INFO_SKIP(SubShape) |
| 510 | POPULATE_PROFILE_INFO_SKIP(Yield) |
| 511 | POPULATE_PROFILE_INFO_SKIP(If) |
| 512 | POPULATE_PROFILE_INFO_SKIP(While) |
| 513 | |
| 514 | return failure(); |
| 515 | } |
| 516 | |
| 517 | //===----------------------------------------------------------------------===// |
| 518 | // Tosa Profile And Extension Compliance Checker |
| 519 | //===----------------------------------------------------------------------===// |
| 520 | |
| 521 | template <typename T> |
| 522 | FailureOr<SmallVector<OpComplianceInfo<T>>> |
| 523 | TosaProfileCompliance::getOperatorMatchedEntries(Operation *op) { |
| 524 | const std::string opName = op->getName().getStringRef().str(); |
| 525 | const auto &complianceMap = getProfileComplianceMap<T>(); |
| 526 | const auto it = complianceMap.find(opName); |
| 527 | if (it == complianceMap.end()) |
| 528 | return {}; |
| 529 | |
| 530 | return findMatchedEntries<T>(op, it->second); |
| 531 | } |
| 532 | |
| 533 | template <typename T> |
| 534 | LogicalResult TosaProfileCompliance::checkProfileOrExtension( |
| 535 | Operation *op, const tosa::TargetEnv &targetEnv) { |
| 536 | const auto maybeOpEntries = getOperatorMatchedEntries<T>(op); |
| 537 | if (failed(maybeOpEntries)) |
| 538 | return success(); |
| 539 | |
| 540 | const auto &opEntries = maybeOpEntries.value(); |
| 541 | if (opEntries.size() == 0) { |
| 542 | // No matched restriction found. |
| 543 | return success(); |
| 544 | } |
| 545 | |
| 546 | // Check the profile/extension requirement according to the current target |
| 547 | // profiles/extensions. |
| 548 | const auto isModeAllowed = [&](const OpComplianceInfo<T> &info) -> bool { |
| 549 | if (info.condition == CheckCondition::allOf) |
| 550 | return targetEnv.allowsAllOf(info.mode); |
| 551 | return targetEnv.allowsAnyOf(info.mode); |
| 552 | }; |
| 553 | |
| 554 | // Check the matched op compliance version does not exceed the target |
| 555 | // specification version. |
| 556 | const TosaSpecificationVersion targetVersion{targetEnv.getSpecVersion()}; |
| 557 | const auto isVersionCompatible = |
| 558 | [&targetVersion](const OpComplianceInfo<T> &info) -> bool { |
| 559 | // Assume compatible if there are no versioned type constraints |
| 560 | if (info.operandTypeInfoSet.empty()) |
| 561 | return true; |
| 562 | const TosaSpecificationVersion complianceVersion{ |
| 563 | info.operandTypeInfoSet.front().second}; |
| 564 | return targetVersion.isBackwardsCompatibleWith(baseVersion: complianceVersion); |
| 565 | }; |
| 566 | |
| 567 | for (const auto &info : opEntries) { |
| 568 | // Type constraints exist, check if they are compatible with the current |
| 569 | // target definition, including: profile, extension and specification |
| 570 | // version |
| 571 | if (isModeAllowed(info) && isVersionCompatible(info)) |
| 572 | return success(); |
| 573 | } |
| 574 | |
| 575 | // No valid entry was found, now emit appropriate error message and return |
| 576 | // failure |
| 577 | std::string message; |
| 578 | llvm::raw_string_ostream os(message); |
| 579 | |
| 580 | os << "illegal: " ; |
| 581 | const size_t numOpEntries = opEntries.size(); |
| 582 | for (const auto &[index, info] : llvm::enumerate(opEntries)) { |
| 583 | bool mismatchedVersion = false; |
| 584 | if (!isVersionCompatible(info)) { |
| 585 | mismatchedVersion = true; |
| 586 | os << "requires specification version compatible with " |
| 587 | << stringifyVersion(info.operandTypeInfoSet.front().second) << " (got " |
| 588 | << stringifyVersion(version: targetVersion) << ") " ; |
| 589 | } |
| 590 | |
| 591 | if (!isModeAllowed(info)) { |
| 592 | if (mismatchedVersion) |
| 593 | os << "and " ; |
| 594 | os << "requires " |
| 595 | << (info.condition == CheckCondition::allOf ? "all of " : "any of " ) |
| 596 | << "[" << llvm::join(stringifyProfile<T>(info.mode), ", " ) |
| 597 | << "] profiles/extensions " ; |
| 598 | } |
| 599 | |
| 600 | if (index != numOpEntries - 1) |
| 601 | os << "OR " ; |
| 602 | } |
| 603 | os << "to be specified in the target environment" ; |
| 604 | |
| 605 | return op->emitOpError(message); |
| 606 | } |
| 607 | |
| 608 | LogicalResult |
| 609 | TosaProfileCompliance::checkProfile(Operation *op, |
| 610 | const tosa::TargetEnv &targetEnv) { |
| 611 | return checkProfileOrExtension<Profile>(op, targetEnv); |
| 612 | } |
| 613 | |
| 614 | LogicalResult |
| 615 | TosaProfileCompliance::checkExtension(Operation *op, |
| 616 | const tosa::TargetEnv &targetEnv) { |
| 617 | return checkProfileOrExtension<Extension>(op, targetEnv); |
| 618 | } |
| 619 | |
| 620 | LogicalResult TosaProfileCompliance::checkInvalid(Operation *op) { |
| 621 | const auto maybeProfEntries = getOperatorMatchedEntries<Profile>(op); |
| 622 | const auto maybeExtEntries = getOperatorMatchedEntries<Extension>(op); |
| 623 | if (failed(Result: maybeProfEntries) && failed(Result: maybeExtEntries)) |
| 624 | return success(); |
| 625 | |
| 626 | const bool hasEntry = |
| 627 | (succeeded(Result: maybeProfEntries) && !maybeProfEntries.value().empty()) || |
| 628 | (succeeded(Result: maybeExtEntries) && !maybeExtEntries.value().empty()); |
| 629 | |
| 630 | if (!hasEntry) { |
| 631 | std::string message; |
| 632 | llvm::raw_string_ostream os(message); |
| 633 | os << "illegal: operation operand/result data types did not align with any " |
| 634 | "profile or extension, got (" ; |
| 635 | |
| 636 | ProfileInfoDepot depot(op); |
| 637 | const ArrayRef<TypeInfo> current = depot.getInfo(); |
| 638 | for (const auto &typeInfo : llvm::drop_end(RangeOrContainer: current)) |
| 639 | os << stringifyTypeInfo(typeInfo) << "," ; |
| 640 | os << stringifyTypeInfo(typeInfo: current.back()) << ")" ; |
| 641 | |
| 642 | // avoid polluting the error message output by outputting only |
| 643 | // the best match |
| 644 | const std::string opName = op->getName().getStringRef().str(); |
| 645 | int maxMatches = -1; |
| 646 | ArrayRef<TypeInfo> bestTypeInfo; |
| 647 | const auto searchBestMatch = [&](const auto &map) { |
| 648 | const auto it = map.find(opName); |
| 649 | if (it == map.end()) |
| 650 | return; |
| 651 | for (const auto &complianceInfos : it->second) { |
| 652 | for (const auto &versionedTypeInfos : |
| 653 | complianceInfos.operandTypeInfoSet) { |
| 654 | const ArrayRef<TypeInfo> typeInfos = versionedTypeInfos.first; |
| 655 | if (current.size() != typeInfos.size()) |
| 656 | continue; |
| 657 | const int matches = llvm::count_if( |
| 658 | llvm::zip_equal(t: current, u: typeInfos), [&](const auto zipType) { |
| 659 | return isSameTypeInfo(a: std::get<0>(zipType), |
| 660 | b: std::get<1>(zipType)); |
| 661 | }); |
| 662 | if (matches > maxMatches) { |
| 663 | maxMatches = matches; |
| 664 | bestTypeInfo = typeInfos; |
| 665 | } |
| 666 | } |
| 667 | } |
| 668 | }; |
| 669 | searchBestMatch(getProfileComplianceMap<Profile>()); |
| 670 | searchBestMatch(getProfileComplianceMap<Extension>()); |
| 671 | |
| 672 | os << ", did you mean (" ; |
| 673 | for (const auto &typeInfo : llvm::drop_end(RangeOrContainer&: bestTypeInfo)) |
| 674 | os << stringifyTypeInfo(typeInfo) << "," ; |
| 675 | os << stringifyTypeInfo(typeInfo: bestTypeInfo.back()) << ")? " ; |
| 676 | os << "Otherwise, please refer to the 'supported data types' for '" |
| 677 | << opName << "' in the specification." ; |
| 678 | op->emitOpError(message); |
| 679 | return failure(); |
| 680 | } |
| 681 | |
| 682 | return success(); |
| 683 | } |
| 684 | |
| 685 | // Find the profiles or extensions requirement according to the signature of |
| 686 | // type of the operand list. |
| 687 | template <typename T> |
| 688 | SmallVector<OpComplianceInfo<T>> TosaProfileCompliance::findMatchedEntries( |
| 689 | Operation *op, ArrayRef<OpComplianceInfo<T>> compInfo) { |
| 690 | assert(compInfo.size() != 0 && |
| 691 | "profile-based compliance information is empty" ); |
| 692 | |
| 693 | // If there are no type constraints for the given operation, return the |
| 694 | // matched entries directly so required profile/extension information can |
| 695 | // be checked |
| 696 | SmallVector<OpComplianceInfo<T>> matchedInfos; |
| 697 | if (llvm::all_of(compInfo, [&matchedInfos](const OpComplianceInfo<T> &info) { |
| 698 | if (info.operandTypeInfoSet.empty()) { |
| 699 | matchedInfos.push_back(info); |
| 700 | return true; |
| 701 | } |
| 702 | return false; |
| 703 | })) |
| 704 | return matchedInfos; |
| 705 | |
| 706 | // Get the type information of the operator we're trying to match |
| 707 | ProfileInfoDepot depot(op); |
| 708 | ArrayRef<TypeInfo> present = depot.getInfo(); |
| 709 | if (present.size() == 0) |
| 710 | return {}; |
| 711 | |
| 712 | // Compare 'present' against known type signatures in the compliance metadata |
| 713 | for (const auto &comp : compInfo) { |
| 714 | for (const auto &set : comp.operandTypeInfoSet) { |
| 715 | ArrayRef<TypeInfo> expected = set.first; |
| 716 | // Tensor-list operators can legitimately have multiple valid signatures |
| 717 | // with different operand/result counts, e.g. data-only and data+scale |
| 718 | // forms. Treat those as non-matches instead of asserting. |
| 719 | if (present.size() != expected.size()) |
| 720 | continue; |
| 721 | |
| 722 | bool isFound = true; |
| 723 | // Compare the type signature between the given operation and the |
| 724 | // compliance metadata. |
| 725 | for (size_t j = 0; j < expected.size(); j++) { |
| 726 | if (!isSameTypeInfo(a: present[j], b: expected[j])) { |
| 727 | // Verify the next mode set from the list. |
| 728 | isFound = false; |
| 729 | break; |
| 730 | } |
| 731 | } |
| 732 | |
| 733 | if (isFound == true) { |
| 734 | SmallVector<VersionedTypeInfo> typeInfoSet{set}; |
| 735 | matchedInfos.emplace_back(comp.mode, std::move(typeInfoSet), |
| 736 | comp.condition); |
| 737 | } |
| 738 | } |
| 739 | } |
| 740 | |
| 741 | return matchedInfos; |
| 742 | } |
| 743 | |
| 744 | // Debug utilites. |
| 745 | template <typename T> |
| 746 | SmallVector<StringRef> |
| 747 | TosaProfileCompliance::stringifyProfile(ArrayRef<T> profiles) { |
| 748 | SmallVector<StringRef> debugStrings; |
| 749 | for (const auto &profile : profiles) { |
| 750 | if constexpr (std::is_same_v<T, Profile>) |
| 751 | debugStrings.push_back(Elt: tosa::stringifyProfile(profile)); |
| 752 | else |
| 753 | debugStrings.push_back(Elt: tosa::stringifyExtension(profile)); |
| 754 | } |
| 755 | return debugStrings; |
| 756 | } |
| 757 | |
| 758 | template <typename T> |
| 759 | SmallVector<StringRef> TosaProfileCompliance::stringifyProfile( |
| 760 | const SmallVector<ArrayRef<T>> &profileSet) { |
| 761 | SmallVector<StringRef> debugStrings; |
| 762 | |
| 763 | for (const auto &profiles : profileSet) { |
| 764 | auto tempStrings = stringifyProfile<T>(profiles); |
| 765 | llvm::append_range(debugStrings, tempStrings); |
| 766 | } |
| 767 | |
| 768 | return debugStrings; |
| 769 | } |
| 770 | |
| 771 | llvm::SmallString<32> |
| 772 | TosaProfileCompliance::stringifyTypeInfo(const TypeInfo &typeInfo) { |
| 773 | const auto stringifyScalarTypeInfo = |
| 774 | [](const TypeInfo &typeInfo) -> llvm::SmallString<32> { |
| 775 | if (typeInfo.typeID == mlir::IntegerType::getTypeID()) { |
| 776 | return {"i" + llvm::utostr(X: typeInfo.bitWidth)}; |
| 777 | } |
| 778 | if (typeInfo.typeID == mlir::Float16Type::getTypeID()) { |
| 779 | return {"f16" }; |
| 780 | } else if (typeInfo.typeID == mlir::Float32Type::getTypeID()) { |
| 781 | return {"f32" }; |
| 782 | } else if (typeInfo.typeID == mlir::BFloat16Type::getTypeID()) { |
| 783 | return {"bf16" }; |
| 784 | } else if (typeInfo.typeID == mlir::Float8E4M3FNType::getTypeID()) { |
| 785 | return {"fp8e4m3" }; |
| 786 | } else if (typeInfo.typeID == mlir::Float8E5M2Type::getTypeID()) { |
| 787 | return {"fp8e5m2" }; |
| 788 | } else if (typeInfo.typeID == mlir::Float6E2M3FNType::getTypeID()) { |
| 789 | return {"fp6e2m3" }; |
| 790 | } else if (typeInfo.typeID == mlir::Float6E3M2FNType::getTypeID()) { |
| 791 | return {"fp6e3m2" }; |
| 792 | } else if (typeInfo.typeID == mlir::Float4E2M1FNType::getTypeID()) { |
| 793 | return {"fp4e2m1" }; |
| 794 | } else if (typeInfo.typeID == mlir::Float8E8M0FNUType::getTypeID()) { |
| 795 | return {"fp8e8m0" }; |
| 796 | } else if (typeInfo.typeID == MXI8Type::getTypeID()) { |
| 797 | return {"mxint8" }; |
| 798 | } |
| 799 | llvm_unreachable("unknown type" ); |
| 800 | }; |
| 801 | |
| 802 | if (typeInfo.typeID == tosa::BlockScaledType::getTypeID()) { |
| 803 | TypeInfo valueInfo = {typeInfo.valueTypeID, typeInfo.bitWidth}; |
| 804 | TypeInfo scaleInfo = {typeInfo.scaleTypeID, 8}; |
| 805 | llvm::SmallString<32> result; |
| 806 | llvm::raw_svector_ostream os(result); |
| 807 | os << "bs" |
| 808 | << tosa::BlockShapeAttr::getBlockShapeValue(blockShape: typeInfo.blockShape.value()) |
| 809 | << "_" << stringifyScalarTypeInfo(scaleInfo) << "_" |
| 810 | << stringifyScalarTypeInfo(valueInfo); |
| 811 | return result; |
| 812 | } |
| 813 | |
| 814 | return stringifyScalarTypeInfo(typeInfo); |
| 815 | } |
| 816 | } // namespace tosa |
| 817 | } // namespace mlir |
| 818 | |