| 1 | //===- XeGPUSgToLaneDistribute.cpp - XeGPU SG to Lane Pass ----------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | #include "mlir/Dialect/GPU/IR/GPUDialect.h" |
| 9 | #include "mlir/Dialect/Index/IR/IndexDialect.h" |
| 10 | #include "mlir/Dialect/Math/IR/Math.h" |
| 11 | #include "mlir/Dialect/MemRef/IR/MemRef.h" |
| 12 | #include "mlir/Dialect/SCF/Transforms/Patterns.h" |
| 13 | #include "mlir/Dialect/Utils/IndexingUtils.h" |
| 14 | #include "mlir/Dialect/Vector/IR/VectorOps.h" |
| 15 | #include "mlir/Dialect/XeGPU/IR/XeGPU.h" |
| 16 | #include "mlir/Dialect/XeGPU/Transforms/Passes.h" |
| 17 | #include "mlir/Dialect/XeGPU/Transforms/Transforms.h" |
| 18 | #include "mlir/Dialect/XeGPU/Transforms/XeGPULayoutImpl.h" |
| 19 | #include "mlir/Dialect/XeGPU/Utils/XeGPUUtils.h" |
| 20 | #include "mlir/Dialect/XeGPU/uArch/uArchCommon.h" |
| 21 | #include "mlir/IR/Builders.h" |
| 22 | #include "mlir/IR/BuiltinAttributes.h" |
| 23 | #include "mlir/IR/BuiltinOps.h" |
| 24 | #include "mlir/IR/BuiltinTypes.h" |
| 25 | #include "mlir/IR/MLIRContext.h" |
| 26 | #include "mlir/IR/Operation.h" |
| 27 | #include "mlir/IR/Value.h" |
| 28 | #include "mlir/IR/ValueRange.h" |
| 29 | #include "mlir/Transforms/DialectConversion.h" |
| 30 | #include "llvm/ADT/SetVector.h" |
| 31 | #include "llvm/Support/LogicalResult.h" |
| 32 | #include "llvm/Support/raw_ostream.h" |
| 33 | #include <optional> |
| 34 | |
| 35 | namespace mlir { |
| 36 | namespace xegpu { |
| 37 | #define GEN_PASS_DEF_XEGPUSGTOLANEDISTRIBUTE |
| 38 | #include "mlir/Dialect/XeGPU/Transforms/Passes.h.inc" |
| 39 | } // namespace xegpu |
| 40 | } // namespace mlir |
| 41 | |
| 42 | using namespace mlir; |
| 43 | |
| 44 | #define DEBUG_TYPE "xegpu-sg-to-lane-distribute" |
| 45 | #define DBGS() (llvm::dbgs() << "[" DEBUG_TYPE "]: ") |
| 46 | |
| 47 | namespace { |
| 48 | |
| 49 | /// Casts the given vector value `v` to the expected vector type `expectedTy`. |
| 50 | static Value castValueTo(ConversionPatternRewriter &rewriter, |
| 51 | TypedValue<VectorType> v, VectorType expectedTy) { |
| 52 | // If the type matches, simply return the value itself. |
| 53 | if (v.getType() == expectedTy) |
| 54 | return v; |
| 55 | // If only shape differs, use shape cast. |
| 56 | if (isa<VectorType>(Val: v.getType()) && |
| 57 | v.getType().getNumElements() == expectedTy.getNumElements()) |
| 58 | return vector::ShapeCastOp::create(builder&: rewriter, location: v.getLoc(), result: expectedTy, source: v); |
| 59 | |
| 60 | // Else create an unrealized cast. |
| 61 | auto newOp = UnrealizedConversionCastOp::create(builder&: rewriter, location: v.getLoc(), |
| 62 | resultTypes: expectedTy, operands: ValueRange{v}); |
| 63 | return newOp.getResult(i: 0); |
| 64 | } |
| 65 | |
| 66 | /// A vector::MultiDimReductionOp at subgroup level in expected form if, it has |
| 67 | /// exactly 1 reduction dimension, it had valid result layout attribute, and |
| 68 | /// result type can be distributed to lanes using the layout. |
| 69 | static bool isValidSubgroupMultiReductionOp(vector::MultiDimReductionOp op) { |
| 70 | auto resLayout = xegpu::getTemporaryLayout(operandOrResult: op->getOpResult(idx: 0)); |
| 71 | // If no layout, not valid. |
| 72 | if (!resLayout || !resLayout.isForSubgroup()) |
| 73 | return false; |
| 74 | // Scalar result (e.g., vector<32xf32> to f32) is valid. |
| 75 | if (op.getType().isIntOrFloat()) |
| 76 | return op.getReductionDims().size() == 1; |
| 77 | VectorType resTy = dyn_cast<VectorType>(Val: op.getType()); |
| 78 | if (!resTy) |
| 79 | return false; |
| 80 | // Compute the distributed result vector type based on the layout. |
| 81 | FailureOr<VectorType> resDistTypeOrFailure = |
| 82 | getDistVecTypeBasedOnLaneLayout(layout: resLayout, originalType: resTy); |
| 83 | if (failed(Result: resDistTypeOrFailure)) |
| 84 | return false; |
| 85 | return op.getReductionDims().size() == 1; |
| 86 | } |
| 87 | |
| 88 | /// A vector::MultiDimReductionOp reduces lane-locally when no data is combined |
| 89 | /// across lanes, which holds exactly when the source's lane_layout along the |
| 90 | /// reduction dimension is 1: every reduced element then lives in one lane. |
| 91 | static bool isReductionLaneLocal(vector::MultiDimReductionOp op) { |
| 92 | // Must be valid MultiDimReductionOp. |
| 93 | assert(isValidSubgroupMultiReductionOp(op) && "Expecting a valid subgroup " |
| 94 | "MultiDimReductionOp" ); |
| 95 | auto srcLayout = xegpu::getTemporaryLayout(operandOrResult: op->getOpOperand(idx: 0)); |
| 96 | ArrayRef<int64_t> reductionDims = op.getReductionDims(); |
| 97 | assert(reductionDims.size() == 1 && |
| 98 | "Expecting single reduction dimension for subgroup multi " |
| 99 | "reduction op" ); |
| 100 | int64_t reductionDim = reductionDims[0]; |
| 101 | SmallVector<int64_t> srcLaneLayout = srcLayout.getEffectiveLaneLayoutAsInt(); |
| 102 | assert(reductionDim < static_cast<int64_t>(srcLaneLayout.size()) && |
| 103 | "Expecting a source lane_layout covering the reduction dimension" ); |
| 104 | return srcLaneLayout[reductionDim] == 1; |
| 105 | } |
| 106 | |
| 107 | /// Given a vector type and its distributed vector type, return the list of |
| 108 | /// dimensions that are distributed. |
| 109 | static SmallVector<int64_t> getDistributedDims(VectorType originalType, |
| 110 | VectorType distributedType) { |
| 111 | assert(originalType.getRank() == distributedType.getRank() && |
| 112 | "original and distributed vector types must have the same rank" ); |
| 113 | SmallVector<int64_t> distributedDims; |
| 114 | for (int64_t i = 0; i < originalType.getRank(); ++i) { |
| 115 | if (distributedType.getDimSize(idx: i) != originalType.getDimSize(idx: i)) |
| 116 | distributedDims.push_back(Elt: i); |
| 117 | } |
| 118 | return distributedDims; |
| 119 | } |
| 120 | |
| 121 | /// Distributes a subgroup-level CreateNdDesc op to lane-level CreateNdDesc |
| 122 | /// op. This simply drops the layout attribute from the tensor descriptor type. |
| 123 | struct SgToLaneCreateNdDesc |
| 124 | : public OpConversionPattern<xegpu::CreateNdDescOp> { |
| 125 | using OpConversionPattern<xegpu::CreateNdDescOp>::OpConversionPattern; |
| 126 | |
| 127 | LogicalResult |
| 128 | matchAndRewrite(xegpu::CreateNdDescOp op, OpAdaptor adaptor, |
| 129 | ConversionPatternRewriter &rewriter) const override { |
| 130 | xegpu::TensorDescType resultType = op.getType(); |
| 131 | // If no layout, nothing to do. |
| 132 | if (!resultType.getLayout()) |
| 133 | return failure(); |
| 134 | |
| 135 | auto newOp = xegpu::CreateNdDescOp::create( |
| 136 | builder&: rewriter, location: op.getLoc(), resultTypes: TypeRange{resultType.dropLayouts()}, |
| 137 | operands: op.getOperands(), properties: op.getProperties(), |
| 138 | discardableAttributes: op->getDiscardableAttrDictionary().getValue()); |
| 139 | rewriter.replaceOp(op, newValues: newOp.getResult()); |
| 140 | return success(); |
| 141 | } |
| 142 | }; |
| 143 | |
| 144 | /// Distributes a subgroup-level LoadNd op to lane-level LoadNd op. Output |
| 145 | /// of lane-level LoadNd op is 1D. ShapeCast is added to restore the |
| 146 | /// original rank. |
| 147 | struct SgToLaneLoadNd : public OpConversionPattern<xegpu::LoadNdOp> { |
| 148 | using OpConversionPattern<xegpu::LoadNdOp>::OpConversionPattern; |
| 149 | |
| 150 | LogicalResult |
| 151 | matchAndRewrite(xegpu::LoadNdOp op, OpAdaptor adaptor, |
| 152 | ConversionPatternRewriter &rewriter) const override { |
| 153 | xegpu::DistributeLayoutAttr layout = op.getAnchorLayout(); |
| 154 | // If no layout, nothing to do. |
| 155 | if (!layout) |
| 156 | return failure(); |
| 157 | // Check if the layout attached to the tensor descriptor is same as the |
| 158 | // anchor layout. Otherwise, this is a conflict. |
| 159 | if (op.getTensorDescType().getLayout() != layout) |
| 160 | return rewriter.notifyMatchFailure( |
| 161 | arg&: op, msg: "conflicting layout attributes on tensor descriptor and anchor" ); |
| 162 | const auto *uArch = |
| 163 | xegpu::uArch::getUArch(archName: xegpu::getChipStr(op).value_or(u: "" )); |
| 164 | if (!uArch) |
| 165 | return rewriter.notifyMatchFailure( |
| 166 | arg&: op, msg: "xegpu::LoadNdOp require target attribute attached to " |
| 167 | "determine transpose " |
| 168 | "requirement" ); |
| 169 | auto supportedLaneResultTyOrFailure = |
| 170 | xegpu::getDistributedVectorType(tdescTy: op.getTensorDescType()); |
| 171 | auto expectedLaneResultTyOrFailure = |
| 172 | xegpu::getDistVecTypeBasedOnLaneLayout(layout, originalType: op.getType()); |
| 173 | if (failed(Result: supportedLaneResultTyOrFailure)) |
| 174 | return rewriter.notifyMatchFailure( |
| 175 | arg&: op, msg: "unable to compute the lane vector type for LoadNdOp" ); |
| 176 | if (failed(Result: expectedLaneResultTyOrFailure)) |
| 177 | return rewriter.notifyMatchFailure( |
| 178 | arg&: op, msg: "unable to compute expected lane vector type from lane layout" ); |
| 179 | auto newOp = xegpu::LoadNdOp::create( |
| 180 | builder&: rewriter, location: op.getLoc(), value: supportedLaneResultTyOrFailure.value(), |
| 181 | TensorDesc: adaptor.getTensorDesc(), offsets: op.getMixedOffsets(), packed: op.getPackedAttr(), |
| 182 | transpose: op.getTransposeAttr(), l1_hint: op.getL1HintAttr(), l2_hint: op.getL2HintAttr(), |
| 183 | l3_hint: op.getL3HintAttr(), /**layout**/ layout: nullptr); |
| 184 | // Set the packed attribute if the layout requires it. |
| 185 | newOp.setPacked(xegpu::requirePacked(layout: cast<xegpu::LayoutAttr>(Val&: layout))); |
| 186 | // Set the transpose attribute if the layout requires it. |
| 187 | if (xegpu::requireTranspose(layout: cast<xegpu::LayoutAttr>(Val&: layout), uArch)) |
| 188 | newOp.setTranspose(DenseI64ArrayAttr::get(context: rewriter.getContext(), content: {1, 0})); |
| 189 | rewriter.replaceOp(op, newValues: castValueTo(rewriter, v: newOp.getResult(), |
| 190 | expectedTy: expectedLaneResultTyOrFailure.value())); |
| 191 | return success(); |
| 192 | } |
| 193 | }; |
| 194 | |
| 195 | /// Distributes a subgroup-level StoreNd op to lane-level StoreNd op. Stored |
| 196 | /// value in lane-level StoreNd op is 1D. ShapeCast is added to cast the |
| 197 | /// incoming value to 1D. |
| 198 | struct SgToLaneStoreNd : public OpConversionPattern<xegpu::StoreNdOp> { |
| 199 | using OpConversionPattern<xegpu::StoreNdOp>::OpConversionPattern; |
| 200 | |
| 201 | LogicalResult |
| 202 | matchAndRewrite(xegpu::StoreNdOp op, OpAdaptor adaptor, |
| 203 | ConversionPatternRewriter &rewriter) const override { |
| 204 | xegpu::DistributeLayoutAttr layout = op.getAnchorLayout(); |
| 205 | // If no layout, nothing to do. |
| 206 | if (!layout) |
| 207 | return failure(); |
| 208 | // Check if the layout attached to the tensor descriptor and value layout is |
| 209 | // same as the anchor layout. Otherwise, this is a conflict. |
| 210 | if (op.getTensorDescType().getLayout() != layout) |
| 211 | return rewriter.notifyMatchFailure( |
| 212 | arg&: op, msg: "conflicting layout attributes on tensor descriptor and anchor" ); |
| 213 | auto valueLayout = xegpu::getDistributeLayoutAttr(opr: op->getOpOperand(idx: 0)); |
| 214 | if (valueLayout != layout) |
| 215 | return rewriter.notifyMatchFailure( |
| 216 | arg&: op, msg: "conflicting layout attributes on value and anchor" ); |
| 217 | auto supportedLaneValueTyOrFailure = |
| 218 | xegpu::getDistributedVectorType(tdescTy: op.getTensorDescType()); |
| 219 | if (failed(Result: supportedLaneValueTyOrFailure)) |
| 220 | return rewriter.notifyMatchFailure( |
| 221 | arg&: op, |
| 222 | msg: "unable to compute lane vector type for StoreNdOp value from tensor " |
| 223 | "descriptor" ); |
| 224 | |
| 225 | xegpu::StoreNdOp::create( |
| 226 | builder&: rewriter, location: op.getLoc(), |
| 227 | value: castValueTo(rewriter, v: cast<TypedValue<VectorType>>(Val: adaptor.getValue()), |
| 228 | expectedTy: supportedLaneValueTyOrFailure.value()), |
| 229 | TensorDesc: adaptor.getTensorDesc(), offsets: op.getMixedOffsets(), l1_hint: op.getL1HintAttr(), |
| 230 | l2_hint: op.getL2HintAttr(), l3_hint: op.getL3HintAttr(), /**layout**/ layout: nullptr); |
| 231 | rewriter.eraseOp(op); |
| 232 | return success(); |
| 233 | } |
| 234 | }; |
| 235 | |
| 236 | /// Distributes a subgroup-level Dpas op to lane-level Dpas op. All inpputs |
| 237 | /// and output of lane-level Dpas op are 1D. Necessary casts are added to |
| 238 | /// convert the inputs and output to/from 1D. |
| 239 | struct SgToLaneDpas : public OpConversionPattern<xegpu::DpasOp> { |
| 240 | using OpConversionPattern<xegpu::DpasOp>::OpConversionPattern; |
| 241 | |
| 242 | LogicalResult |
| 243 | matchAndRewrite(xegpu::DpasOp op, OpAdaptor adaptor, |
| 244 | ConversionPatternRewriter &rewriter) const override { |
| 245 | // Check if the op has A, B and CD layouts attached. |
| 246 | auto layoutA = cast<xegpu::LayoutAttr>(Val: op.getLayoutAAttr()); |
| 247 | auto layoutB = cast<xegpu::LayoutAttr>(Val: op.getLayoutBAttr()); |
| 248 | auto layoutCd = cast<xegpu::LayoutAttr>(Val: op.getLayoutCdAttr()); |
| 249 | if (!layoutA || !layoutB || !layoutCd) |
| 250 | return failure(); |
| 251 | auto laneResultTyOrFailure = |
| 252 | xegpu::getDistributedVectorType(originalType: op.getType(), layout: layoutCd); |
| 253 | auto laneATypeOrFailure = |
| 254 | xegpu::getDistributedVectorType(originalType: op.getLhs().getType(), layout: layoutA); |
| 255 | auto laneBTypeOrFailure = |
| 256 | xegpu::getDistributedVectorType(originalType: op.getRhs().getType(), layout: layoutB); |
| 257 | auto expectedLaneResultTyOrFailure = |
| 258 | xegpu::getDistVecTypeBasedOnLaneLayout(layout: layoutCd, originalType: op.getType()); |
| 259 | if (failed(Result: laneResultTyOrFailure) || failed(Result: laneATypeOrFailure) || |
| 260 | failed(Result: laneBTypeOrFailure)) |
| 261 | return rewriter.notifyMatchFailure( |
| 262 | arg&: op, msg: "failed to calculate supported lane vector types for DpasOp " |
| 263 | "from layouts" ); |
| 264 | if (failed(Result: expectedLaneResultTyOrFailure)) |
| 265 | return rewriter.notifyMatchFailure( |
| 266 | arg&: op, msg: "unable to compute expected lane vector type for DpasOp from " |
| 267 | "lane layout" ); |
| 268 | |
| 269 | // Validate bit widths match uArch packed format requirements |
| 270 | const auto *uArch = |
| 271 | xegpu::uArch::getUArch(archName: xegpu::getChipStr(op).value_or(u: "" )); |
| 272 | if (uArch) { |
| 273 | const auto *uArchInstruction = |
| 274 | dyn_cast<xegpu::uArch::SubgroupMatrixMultiplyAcc>( |
| 275 | Val: uArch->getInstruction( |
| 276 | instKind: xegpu::uArch::InstructionKind::SubgroupMatrixMultiplyAcc)); |
| 277 | if (uArchInstruction) { |
| 278 | auto laneAType = laneATypeOrFailure.value(); |
| 279 | auto laneBType = laneBTypeOrFailure.value(); |
| 280 | // Calculate total packed bit width = element bit width * vector size |
| 281 | unsigned aPackedBitWidth = |
| 282 | laneAType.getElementTypeBitWidth() * laneAType.getNumElements(); |
| 283 | unsigned bPackedBitWidth = |
| 284 | laneBType.getElementTypeBitWidth() * laneBType.getNumElements(); |
| 285 | unsigned expectedABitSize = uArchInstruction->getPackedFormatBitSizeA(); |
| 286 | unsigned expectedBBitSize = uArchInstruction->getPackedFormatBitSizeB(); |
| 287 | |
| 288 | if (aPackedBitWidth % expectedABitSize != 0) |
| 289 | return rewriter.notifyMatchFailure( |
| 290 | arg&: op, |
| 291 | msg: "A operand packed bit width must be a multiple of uArch packed " |
| 292 | "format requirement" ); |
| 293 | if (bPackedBitWidth % expectedBBitSize != 0) |
| 294 | return rewriter.notifyMatchFailure( |
| 295 | arg&: op, |
| 296 | msg: "B operand packed bit width must be a multiple of uArch packed " |
| 297 | "format requirement" ); |
| 298 | } |
| 299 | } |
| 300 | |
| 301 | auto newOp = xegpu::DpasOp::create( |
| 302 | builder&: rewriter, location: op->getLoc(), result: laneResultTyOrFailure.value(), |
| 303 | lhs: castValueTo(rewriter, v: cast<TypedValue<VectorType>>(Val: adaptor.getLhs()), |
| 304 | expectedTy: laneATypeOrFailure.value()), |
| 305 | rhs: castValueTo(rewriter, v: cast<TypedValue<VectorType>>(Val: adaptor.getRhs()), |
| 306 | expectedTy: laneBTypeOrFailure.value()), |
| 307 | acc: castValueTo(rewriter, v: cast<TypedValue<VectorType>>(Val: adaptor.getAcc()), |
| 308 | expectedTy: laneResultTyOrFailure.value()), |
| 309 | /** layoutA**/ layout_a: nullptr, |
| 310 | /** layoutB**/ layout_b: nullptr, /** layoutCd**/ layout_cd: nullptr); |
| 311 | // Explicitly set the new types to enable correct type materializations. |
| 312 | rewriter.replaceOp(op, newValues: castValueTo(rewriter, v: newOp.getResult(), |
| 313 | expectedTy: expectedLaneResultTyOrFailure.value())); |
| 314 | return success(); |
| 315 | } |
| 316 | }; |
| 317 | |
| 318 | /// Distributes elementwise ops to lane-level elementwise ops. This |
| 319 | /// currently handles elementwise ops with single result only. |
| 320 | struct SgToLaneElementWise : public ConversionPattern { |
| 321 | SgToLaneElementWise(TypeConverter &typeConverter, MLIRContext *ctx) |
| 322 | : ConversionPattern(MatchAnyOpTypeTag(), /*benefit=*/1, ctx) {} |
| 323 | |
| 324 | LogicalResult |
| 325 | matchAndRewrite(Operation *op, ArrayRef<Value> operands, |
| 326 | ConversionPatternRewriter &rewriter) const override { |
| 327 | // Only match ops with elementwise trait and single result. |
| 328 | if (!OpTrait::hasElementwiseMappableTraits(op) || op->getNumResults() != 1) |
| 329 | return failure(); |
| 330 | |
| 331 | auto resultType = dyn_cast<VectorType>(Val: op->getResult(idx: 0).getType()); |
| 332 | if (!resultType) |
| 333 | return rewriter.notifyMatchFailure( |
| 334 | arg&: op, msg: "operation result is not a vector type" ); |
| 335 | |
| 336 | xegpu::DistributeLayoutAttr layout = |
| 337 | xegpu::getTemporaryLayout(operandOrResult: llvm::cast<OpResult>(Val: op->getResult(idx: 0))); |
| 338 | if (!layout || !layout.isForSubgroup()) |
| 339 | return rewriter.notifyMatchFailure( |
| 340 | arg&: op, msg: "operation result does not have subgroup distribute layout" ); |
| 341 | |
| 342 | auto laneShapeOrFailure = |
| 343 | xegpu::getDistVecTypeBasedOnLaneLayout(layout, originalType: resultType); |
| 344 | |
| 345 | if (failed(Result: laneShapeOrFailure)) |
| 346 | return rewriter.notifyMatchFailure( |
| 347 | arg&: op, msg: "unable to compute lane vector type from the layout" ); |
| 348 | |
| 349 | VectorType newResultType = laneShapeOrFailure.value(); |
| 350 | OperationState state(op->getLoc(), op->getName()); |
| 351 | state.addOperands(newOperands: operands); |
| 352 | state.addTypes(newTypes: newResultType); |
| 353 | // Copy all attributes except for DistributeLayoutAttr. |
| 354 | for (auto attr : op->getDiscardableAttrDictionary().getValue()) { |
| 355 | if (!isa<xegpu::DistributeLayoutAttr>(Val: attr.getValue())) |
| 356 | state.addAttribute(name: attr.getName(), attr: attr.getValue()); |
| 357 | } |
| 358 | state.propertiesAttr = op->getPropertiesAsAttribute(); |
| 359 | Operation *newOp = rewriter.create(state); |
| 360 | |
| 361 | rewriter.replaceOp(op, newValues: newOp->getResult(idx: 0)); |
| 362 | return success(); |
| 363 | } |
| 364 | }; |
| 365 | |
| 366 | /// Distributes a subgroup-level arith ConstantOp to lane-level arith |
| 367 | /// ConstantOp. |
| 368 | /// |
| 369 | /// Splat constants are rebuilt with the lane-local vector type. Non-splat |
| 370 | /// constants are distributed by extracting each lane_data-sized block from |
| 371 | /// the full constant and inserting it at the correct position in the |
| 372 | /// distributed vector using insert_strided_slice. |
| 373 | struct SgToLaneArithConstant : public OpConversionPattern<arith::ConstantOp> { |
| 374 | using OpConversionPattern<arith::ConstantOp>::OpConversionPattern; |
| 375 | |
| 376 | LogicalResult |
| 377 | matchAndRewrite(arith::ConstantOp op, OpAdaptor adaptor, |
| 378 | ConversionPatternRewriter &rewriter) const override { |
| 379 | auto resultType = dyn_cast<VectorType>(Val: op.getType()); |
| 380 | if (!resultType) |
| 381 | return failure(); |
| 382 | |
| 383 | // Only handle dense vector constants. |
| 384 | auto denseAttr = dyn_cast<DenseElementsAttr>(Val: op.getValue()); |
| 385 | if (!denseAttr) |
| 386 | return rewriter.notifyMatchFailure( |
| 387 | arg&: op, msg: "only dense vector constants are supported" ); |
| 388 | |
| 389 | xegpu::DistributeLayoutAttr layout = |
| 390 | xegpu::getTemporaryLayout(operandOrResult: llvm::cast<OpResult>(Val: op.getResult())); |
| 391 | if (!layout || !layout.isForSubgroup()) |
| 392 | return rewriter.notifyMatchFailure( |
| 393 | arg&: op, msg: "operation result does not have subgroup distribute layout" ); |
| 394 | |
| 395 | auto laneShapeOrFailure = |
| 396 | xegpu::getDistVecTypeBasedOnLaneLayout(layout, originalType: resultType); |
| 397 | |
| 398 | if (failed(Result: laneShapeOrFailure)) |
| 399 | return rewriter.notifyMatchFailure( |
| 400 | arg&: op, msg: "unable to compute lane vector type from the layout" ); |
| 401 | |
| 402 | VectorType newResultType = laneShapeOrFailure.value(); |
| 403 | Location loc = op.getLoc(); |
| 404 | |
| 405 | // Splat constants: every lane gets the same value, so just rebuild the |
| 406 | // splat with the distributed type. |
| 407 | if (denseAttr.isSplat()) { |
| 408 | auto scalarValue = denseAttr.getSplatValue<Attribute>(); |
| 409 | auto newDenseAttr = DenseElementsAttr::get(type: newResultType, values: scalarValue); |
| 410 | auto newOp = |
| 411 | arith::ConstantOp::create(builder&: rewriter, location: loc, result: newResultType, value: newDenseAttr); |
| 412 | rewriter.replaceOp(op, newValues: newOp.getResult()); |
| 413 | return success(); |
| 414 | } |
| 415 | |
| 416 | // Non-splat constants: each lane extracts the elements it owns from the |
| 417 | // full constant using the distributed coordinates from the layout. |
| 418 | auto fullConst = |
| 419 | arith::ConstantOp::create(builder&: rewriter, location: loc, result: resultType, value: denseAttr); |
| 420 | |
| 421 | Value laneId = gpu::LaneIdOp::create(builder&: rewriter, location: loc, result: rewriter.getIndexType(), |
| 422 | /*upperBound=*/upper_bound: mlir::IntegerAttr()); |
| 423 | auto maybeCoordsVec = layout.computeDistributedCoords( |
| 424 | builder&: rewriter, loc, linearId: laneId, shape: resultType.getShape()); |
| 425 | if (failed(Result: maybeCoordsVec)) |
| 426 | return rewriter.notifyMatchFailure( |
| 427 | arg&: op, msg: "failed to compute distributed coordinates from layout" ); |
| 428 | |
| 429 | SmallVector<SmallVector<Value>> coordsVec = maybeCoordsVec.value(); |
| 430 | SmallVector<int64_t> laneData = layout.getEffectiveLaneDataAsInt(); |
| 431 | ArrayRef<int64_t> distShape = newResultType.getShape(); |
| 432 | int64_t rank = newResultType.getRank(); |
| 433 | |
| 434 | // Each lane owns one lane_data-sized block per distribution unit. |
| 435 | // computeDistributedCoords returns those block starts in row-major order |
| 436 | // over the block grid (distShape / laneData). |
| 437 | SmallVector<int64_t> blockGridShape(rank); |
| 438 | for (int64_t d = 0; d < rank; d++) |
| 439 | blockGridShape[d] = distShape[d] / laneData[d]; |
| 440 | SmallVector<int64_t> blockGridStrides = computeStrides(sizes: blockGridShape); |
| 441 | |
| 442 | auto blockType = VectorType::get(shape: laneData, elementType: newResultType.getElementType()); |
| 443 | SmallVector<int64_t> unitTile(rank, 1); |
| 444 | SmallVector<int64_t> strides(rank, 1); |
| 445 | |
| 446 | Value result = arith::ConstantOp::create( |
| 447 | builder&: rewriter, location: loc, result: newResultType, value: rewriter.getZeroAttr(type: newResultType)); |
| 448 | |
| 449 | for (auto [blockIdx, blockStart] : llvm::enumerate(First&: coordsVec)) { |
| 450 | // Gather the block's elements from the full constant. The block start is |
| 451 | // lane-dynamic, so extract element-by-element (row-major over lane_data) |
| 452 | // instead. |
| 453 | SmallVector<Value> blockElems; |
| 454 | for (SmallVector<int64_t> off : |
| 455 | StaticTileOffsetRange(laneData, unitTile)) { |
| 456 | SmallVector<OpFoldResult> pos(rank); |
| 457 | for (int64_t d = 0; d < rank; d++) |
| 458 | pos[d] = getAsOpFoldResult(val: arith::AddIOp::create( |
| 459 | builder&: rewriter, location: loc, lhs: blockStart[d], |
| 460 | rhs: arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: off[d]))); |
| 461 | blockElems.push_back(Elt: vector::ExtractOp::create( |
| 462 | builder&: rewriter, location: loc, source: fullConst.getResult(), position: pos)); |
| 463 | } |
| 464 | |
| 465 | // Rebuild the block keeping its lane_data shape, then place it with |
| 466 | // insert_strided_slice so the block keeps its orientation in the |
| 467 | // distributed vector (e.g. a [2, 1] block stays a vertical 2x1 slice). |
| 468 | Value block = |
| 469 | vector::FromElementsOp::create(builder&: rewriter, location: loc, result: blockType, elements: blockElems); |
| 470 | SmallVector<int64_t> blockGridPos = |
| 471 | delinearize(linearIndex: blockIdx, strides: blockGridStrides); |
| 472 | SmallVector<int64_t> offsets(rank); |
| 473 | for (int64_t d = 0; d < rank; d++) |
| 474 | offsets[d] = blockGridPos[d] * laneData[d]; |
| 475 | result = vector::InsertStridedSliceOp::create(builder&: rewriter, location: loc, valueToStore: block, |
| 476 | dest: result, offsets, strides); |
| 477 | } |
| 478 | |
| 479 | rewriter.replaceOp(op, newValues: result); |
| 480 | return success(); |
| 481 | } |
| 482 | }; |
| 483 | |
| 484 | /// Distributes a subgroup-level PrefetchNd op to lane-level PrefetchNd op. |
| 485 | struct SgToLanePrefetchNd : public OpConversionPattern<xegpu::PrefetchNdOp> { |
| 486 | using OpConversionPattern<xegpu::PrefetchNdOp>::OpConversionPattern; |
| 487 | |
| 488 | LogicalResult |
| 489 | matchAndRewrite(xegpu::PrefetchNdOp op, OpAdaptor adaptor, |
| 490 | ConversionPatternRewriter &rewriter) const override { |
| 491 | xegpu::DistributeLayoutAttr layout = op.getAnchorLayout(); |
| 492 | // If no layout, nothing to do. |
| 493 | if (!layout) |
| 494 | return failure(); |
| 495 | |
| 496 | xegpu::PrefetchNdOp::create(builder&: rewriter, location: op.getLoc(), TensorDesc: adaptor.getTensorDesc(), |
| 497 | offsets: op.getMixedOffsets(), l1_hint: op.getL1HintAttr(), |
| 498 | l2_hint: op.getL2HintAttr(), l3_hint: op.getL3HintAttr(), |
| 499 | /**layout**/ layout: nullptr); |
| 500 | rewriter.eraseOp(op); |
| 501 | return success(); |
| 502 | } |
| 503 | }; |
| 504 | |
| 505 | /// Distributes a subgroup-level LoadGather (xegpu.load) op to lane-level. |
| 506 | /// |
| 507 | /// Example 1 (1D): |
| 508 | /// layout = #xegpu.layout<lane_layout = [16], lane_data = [1]> |
| 509 | /// %mask = producer_op : vector<16xi1> |
| 510 | /// %offset = producer_op : vector<16xindex> |
| 511 | /// %0 = xegpu.load %src[%offset], %mask : memref<256xf16>, |
| 512 | /// vector<16xindex>, vector<16xi1> -> vector<16xf16> |
| 513 | /// Distributed to: |
| 514 | /// %mask = producer_op : vector<1xi1> |
| 515 | /// %offset = producer_op : vector<1xindex> |
| 516 | /// %0 = xegpu.load %src[%offset], %mask : memref<256xf16>, |
| 517 | /// vector<1xindex>, vector<1xi1> -> vector<1xf16> |
| 518 | /// |
| 519 | /// Example 2 (3D with leading unit dims): |
| 520 | /// layout = #xegpu.layout<lane_layout = [1, 1, 16], lane_data = [1, 1, 1]> |
| 521 | /// %mask = producer_op : vector<1x1x16xi1> |
| 522 | /// %offset = producer_op : vector<1x1x16xindex> |
| 523 | /// %0 = xegpu.load %src[%offset], %mask : memref<256xf16>, |
| 524 | /// vector<1x1x16xindex>, vector<1x1x16xi1> -> vector<1x1x16xf16> |
| 525 | /// Distributed to: |
| 526 | /// %mask = producer_op : vector<1x1x1xi1> |
| 527 | /// %offset = producer_op : vector<1x1x1xindex> |
| 528 | /// %0 = xegpu.load %src[%offset], %mask : memref<256xf16>, |
| 529 | /// vector<1xindex>, vector<1xi1> -> vector<1xf16> |
| 530 | struct SgToLaneLoadGather : public OpConversionPattern<xegpu::LoadGatherOp> { |
| 531 | using OpConversionPattern<xegpu::LoadGatherOp>::OpConversionPattern; |
| 532 | |
| 533 | LogicalResult |
| 534 | matchAndRewrite(xegpu::LoadGatherOp op, OpAdaptor adaptor, |
| 535 | ConversionPatternRewriter &rewriter) const override { |
| 536 | xegpu::DistributeLayoutAttr layout = op.getAnchorLayout(); |
| 537 | if (!layout) |
| 538 | return failure(); |
| 539 | |
| 540 | VectorType origResultTy = op.getValueType(); |
| 541 | if (!origResultTy) |
| 542 | return failure(); |
| 543 | |
| 544 | // Check that leading dimensions are unit. |
| 545 | int effectiveVecRank = 1; |
| 546 | ArrayRef<int64_t> shape = origResultTy.getShape(); |
| 547 | if (llvm::any_of( |
| 548 | Range: shape.take_front(N: origResultTy.getRank() - effectiveVecRank), |
| 549 | P: [](int64_t d) { return d != 1; })) |
| 550 | return rewriter.notifyMatchFailure( |
| 551 | arg&: op, msg: "Only unit dimensions allowed for the leading " |
| 552 | "dimensions of the load vector!" ); |
| 553 | |
| 554 | auto distResultTyOrFailure = |
| 555 | xegpu::getDistVecTypeBasedOnLaneLayout(layout, originalType: origResultTy); |
| 556 | if (failed(Result: distResultTyOrFailure)) |
| 557 | return rewriter.notifyMatchFailure( |
| 558 | arg&: op, msg: "unable to compute expected lane vector type from lane layout" ); |
| 559 | |
| 560 | VectorType distResultTy = distResultTyOrFailure.value(); |
| 561 | VectorType distResultTy1D = VectorType::get(shape: {distResultTy.getNumElements()}, |
| 562 | elementType: distResultTy.getElementType()); |
| 563 | |
| 564 | // Flatten offsets and mask to 1D to match the 1D result type. |
| 565 | Value distOffsets = adaptor.getOffsets(); |
| 566 | auto distOffsetsTy = cast<VectorType>(Val: distOffsets.getType()); |
| 567 | VectorType offsetsTy1D = VectorType::get(shape: {distOffsetsTy.getNumElements()}, |
| 568 | elementType: distOffsetsTy.getElementType()); |
| 569 | distOffsets = castValueTo( |
| 570 | rewriter, v: cast<TypedValue<VectorType>>(Val&: distOffsets), expectedTy: offsetsTy1D); |
| 571 | |
| 572 | Value distMask = adaptor.getMask(); |
| 573 | auto distMaskTy = cast<VectorType>(Val: distMask.getType()); |
| 574 | VectorType maskTy1D = VectorType::get(shape: {distMaskTy.getNumElements()}, |
| 575 | elementType: distMaskTy.getElementType()); |
| 576 | distMask = |
| 577 | castValueTo(rewriter, v: cast<TypedValue<VectorType>>(Val&: distMask), expectedTy: maskTy1D); |
| 578 | |
| 579 | Value distSource = adaptor.getSource(); |
| 580 | auto newOp = xegpu::LoadGatherOp::create( |
| 581 | builder&: rewriter, location: op.getLoc(), value: distResultTy1D, source: distSource, offsets: distOffsets, |
| 582 | mask: distMask, l1_hint: op.getL1HintAttr(), l2_hint: op.getL2HintAttr(), l3_hint: op.getL3HintAttr(), |
| 583 | /*layout=*/nullptr, /*contiguity=*/nullptr); |
| 584 | |
| 585 | Value result = newOp->getResult(idx: 0); |
| 586 | if (distResultTy1D != distResultTy) |
| 587 | result = castValueTo(rewriter, v: cast<TypedValue<VectorType>>(Val&: result), |
| 588 | expectedTy: distResultTy); |
| 589 | rewriter.replaceOp(op, newValues: result); |
| 590 | return success(); |
| 591 | } |
| 592 | }; |
| 593 | |
| 594 | /// This pattern distributes a subgroup-level vector.reduction op to |
| 595 | /// lane-level. This require shuffling the data across the lanes (using |
| 596 | /// gpu::ShuffleOp) and reducing in stages until all lanes have the final |
| 597 | /// result. |
| 598 | struct SgToLaneVectorReduction |
| 599 | : public OpConversionPattern<vector::ReductionOp> { |
| 600 | using OpConversionPattern<vector::ReductionOp>::OpConversionPattern; |
| 601 | |
| 602 | LogicalResult |
| 603 | matchAndRewrite(vector::ReductionOp op, OpAdaptor adaptor, |
| 604 | ConversionPatternRewriter &rewriter) const override { |
| 605 | auto layout = xegpu::getDistributeLayoutAttr(value: op.getVector()); |
| 606 | |
| 607 | // If no layout, nothing to do. |
| 608 | if (!layout || !layout.isForSubgroup()) |
| 609 | return failure(); |
| 610 | |
| 611 | VectorType srcVecType = op.getSourceVectorType(); |
| 612 | // Only rank 1 vectors supported. |
| 613 | if (srcVecType.getRank() != 1) |
| 614 | return rewriter.notifyMatchFailure( |
| 615 | arg&: op, msg: "Only rank 1 reductions can be distributed." ); |
| 616 | // Lane layout must have the same rank as the vector. |
| 617 | if (layout.getRank() != srcVecType.getRank()) |
| 618 | return rewriter.notifyMatchFailure( |
| 619 | arg&: op, msg: "Layout rank does not match vector rank." ); |
| 620 | |
| 621 | // Get the subgroup size from the layout. |
| 622 | int64_t sgSize = layout.getEffectiveLaneLayoutAsInt()[0]; |
| 623 | const auto *uArch = |
| 624 | xegpu::uArch::getUArch(archName: xegpu::getChipStr(op).value_or(u: "" )); |
| 625 | if (!uArch) |
| 626 | return rewriter.notifyMatchFailure( |
| 627 | arg&: op, msg: "xegpu::ReductionOp require target attribute attached to " |
| 628 | "determine subgroup size" ); |
| 629 | |
| 630 | // Only subgroup-sized vectors supported. |
| 631 | if (sgSize != uArch->getSubgroupSize() || |
| 632 | srcVecType.getShape()[0] % sgSize != 0) |
| 633 | return rewriter.notifyMatchFailure(arg&: op, |
| 634 | msg: "Invalid layout or reduction vector " |
| 635 | "dimension must match subgroup size." ); |
| 636 | |
| 637 | if (!op.getType().isIntOrFloat()) |
| 638 | return rewriter.notifyMatchFailure( |
| 639 | arg&: op, msg: "Reduction distribution currently only supports floats and " |
| 640 | "integer types." ); |
| 641 | |
| 642 | // Get the distributed vector (per lane portion). |
| 643 | Value laneValVec = adaptor.getVector(); |
| 644 | |
| 645 | // Distribute and reduce across lanes in the subgroup. |
| 646 | Value fullReduce = |
| 647 | xegpu::subgroupReduction(loc: op.getLoc(), builder&: rewriter, input: laneValVec, |
| 648 | kind: op.getKind(), size: sgSize, fastmath: op.getFastmathAttr()); |
| 649 | |
| 650 | // If there's an accumulator, combine it with the reduced value. |
| 651 | if (adaptor.getAcc()) |
| 652 | fullReduce = vector::makeArithReduction( |
| 653 | b&: rewriter, loc: op.getLoc(), kind: op.getKind(), v1: fullReduce, acc: adaptor.getAcc(), |
| 654 | fastmath: op.getFastmathAttr()); |
| 655 | |
| 656 | rewriter.replaceOp(op, newValues: fullReduce); |
| 657 | return success(); |
| 658 | } |
| 659 | }; |
| 660 | |
| 661 | /// This pattern distributes a subgroup-level vector.multi_reduction op to |
| 662 | /// lane-level only if the reduction is lane-local. This means that |
| 663 | /// reduction dimension is not distributed to lanes and each lane does its own |
| 664 | /// local reduction. |
| 665 | struct SgToLaneMultiDimReduction |
| 666 | : public OpConversionPattern<vector::MultiDimReductionOp> { |
| 667 | using OpConversionPattern<vector::MultiDimReductionOp>::OpConversionPattern; |
| 668 | |
| 669 | LogicalResult |
| 670 | matchAndRewrite(vector::MultiDimReductionOp op, OpAdaptor adaptor, |
| 671 | ConversionPatternRewriter &rewriter) const override { |
| 672 | Value result; |
| 673 | ArrayRef<int64_t> reductionDims = op.getReductionDims(); |
| 674 | assert(reductionDims.size() == 1 && |
| 675 | "Expecting single reduction dimension for subgroup multi " |
| 676 | "reduction op" ); |
| 677 | // For rank > 2, ensure leading dimensions are unit. |
| 678 | VectorType sourceType = op.getSourceVectorType(); |
| 679 | int64_t rank = sourceType.getRank(); |
| 680 | if (rank > 2) { |
| 681 | ArrayRef<int64_t> shape = sourceType.getShape(); |
| 682 | if (llvm::any_of(Range: shape.take_front(N: rank - 2), |
| 683 | P: [](int64_t d) { return d != 1; })) |
| 684 | return rewriter.notifyMatchFailure( |
| 685 | arg&: op, msg: "only unit leading dimensions are supported for " |
| 686 | "multi_reduction with rank > 2" ); |
| 687 | } |
| 688 | // Handle scalar result: full reduction of a distributed vector to a |
| 689 | // scalar. First do a local vector reduction, then cross-lane shuffles. |
| 690 | if (op.getType().isIntOrFloat()) { |
| 691 | auto reductionDim = reductionDims[0]; |
| 692 | VectorType origSourceType = op.getSourceVectorType(); |
| 693 | int64_t reductionDimSize = origSourceType.getShape()[reductionDim]; |
| 694 | // Local reduction to scalar, then cross-lane butterfly shuffles. |
| 695 | result = xegpu::subgroupReduction(loc: op.getLoc(), builder&: rewriter, |
| 696 | input: adaptor.getSource(), kind: op.getKind(), |
| 697 | size: reductionDimSize, fastmath: op.getFastmathAttr()); |
| 698 | // Combine with accumulator if present. |
| 699 | if (adaptor.getAcc()) |
| 700 | result = vector::makeArithReduction(b&: rewriter, loc: op.getLoc(), kind: op.getKind(), |
| 701 | v1: result, acc: adaptor.getAcc(), |
| 702 | fastmath: op.getFastmathAttr()); |
| 703 | } else if (isReductionLaneLocal(op)) { |
| 704 | // For lane-local reduction, lower to a sequence of vector.reduction ops |
| 705 | // over 1D slices extracted from the distributed source vector. This is |
| 706 | // required so we dont have 2D source vectors at xegpu-linearize. |
| 707 | auto reductionDim = reductionDims[0]; |
| 708 | result = xegpu::lowerToVectorReductions( |
| 709 | src: cast<TypedValue<VectorType>>(Val: adaptor.getSource()), |
| 710 | acc: cast<TypedValue<VectorType>>(Val: adaptor.getAcc()), kind: op.getKind(), |
| 711 | reductionDim, loc: op.getLoc(), rewriter, fastmath: op.getFastmathAttr()); |
| 712 | } else { |
| 713 | auto reductionDim = reductionDims[0]; |
| 714 | VectorType sourceType = op.getSourceVectorType(); |
| 715 | int64_t reductionDimSize = sourceType.getShape()[reductionDim]; |
| 716 | result = xegpu::lowerCrossLaneReductionToShuffles( |
| 717 | src: cast<TypedValue<VectorType>>(Val: adaptor.getSource()), |
| 718 | acc: cast<TypedValue<VectorType>>(Val: adaptor.getAcc()), kind: op.getKind(), |
| 719 | reductionDim, reductionSize: reductionDimSize, loc: op.getLoc(), rewriter, |
| 720 | fastmath: op.getFastmathAttr()); |
| 721 | } |
| 722 | rewriter.replaceOp(op, newValues: result); |
| 723 | return success(); |
| 724 | } |
| 725 | }; |
| 726 | |
| 727 | /// Helper to compute distributed coordinates for matrix ops. |
| 728 | /// When not using subgroup_block_io, each lane computes its own |
| 729 | /// coordinates based on the layout and lane ID. |
| 730 | static SmallVector<Value> computeDistributedCoordsForMatrixOp( |
| 731 | ConversionPatternRewriter &rewriter, Location loc, |
| 732 | xegpu::DistributeLayoutAttr layout, ArrayRef<int64_t> payloadShape, |
| 733 | ValueRange origOffsets) { |
| 734 | Value laneId = gpu::LaneIdOp::create(builder&: rewriter, location: loc, result: rewriter.getIndexType(), |
| 735 | /*upperBound=*/upper_bound: mlir::IntegerAttr()); |
| 736 | auto maybeCoords = |
| 737 | layout.computeDistributedCoords(builder&: rewriter, loc, linearId: laneId, shape: payloadShape); |
| 738 | if (failed(Result: maybeCoords)) |
| 739 | return {}; |
| 740 | assert(maybeCoords.value().size() == 1 && |
| 741 | "Expected one set of distributed offsets" ); |
| 742 | SmallVector<OpFoldResult> ofrVec = xegpu::addWithRightAligned( |
| 743 | builder&: rewriter, loc, lhs: getAsOpFoldResult(values: maybeCoords.value()[0]), |
| 744 | rhs: getAsOpFoldResult(values: origOffsets)); |
| 745 | return llvm::map_to_vector(C&: ofrVec, F: llvm::CastTo<Value>); |
| 746 | } |
| 747 | |
| 748 | /// This pattern distributes a subgroup-level LoadMatrix op to lane-level. |
| 749 | struct SgToLaneLoadMatrix : public OpConversionPattern<xegpu::LoadMatrixOp> { |
| 750 | using OpConversionPattern<xegpu::LoadMatrixOp>::OpConversionPattern; |
| 751 | |
| 752 | LogicalResult |
| 753 | matchAndRewrite(xegpu::LoadMatrixOp op, OpAdaptor adaptor, |
| 754 | ConversionPatternRewriter &rewriter) const override { |
| 755 | auto layout = op.getLayoutAttr(); |
| 756 | // If no layout, nothing to do. |
| 757 | if (!layout) |
| 758 | return failure(); |
| 759 | |
| 760 | VectorType sgPayloadTy = dyn_cast<VectorType>(Val: op.getResult().getType()); |
| 761 | if (!sgPayloadTy) |
| 762 | return rewriter.notifyMatchFailure( |
| 763 | arg&: op, msg: "the matrix op payload must be a vector type" ); |
| 764 | |
| 765 | auto loc = op.getLoc(); |
| 766 | auto offsets = op.getMixedOffsets(); |
| 767 | if (offsets.empty()) |
| 768 | return rewriter.notifyMatchFailure(arg&: op, msg: "the load op must have offsets" ); |
| 769 | |
| 770 | FailureOr<VectorType> distPayloadTyOrFailure = |
| 771 | getDistVecTypeBasedOnLaneLayout(layout, originalType: sgPayloadTy); |
| 772 | if (failed(Result: distPayloadTyOrFailure)) |
| 773 | return rewriter.notifyMatchFailure( |
| 774 | arg&: op, msg: "Failed to distribute matrix op payload based on layout." ); |
| 775 | |
| 776 | SmallVector<Value> offsetsAsValues = |
| 777 | vector::getAsValues(builder&: rewriter, loc, foldResults: offsets); |
| 778 | |
| 779 | SmallVector<Value> newCoords = offsetsAsValues; |
| 780 | if (!op.getSubgroupBlockIoAttr()) { |
| 781 | newCoords = computeDistributedCoordsForMatrixOp( |
| 782 | rewriter, loc, layout, payloadShape: sgPayloadTy.getShape(), origOffsets: offsetsAsValues); |
| 783 | if (newCoords.empty()) |
| 784 | return rewriter.notifyMatchFailure( |
| 785 | arg&: op, msg: "Failed to compute distributed coordinates." ); |
| 786 | } |
| 787 | |
| 788 | SmallVector<int64_t> newConstOffsets(op.getConstOffsets().size(), |
| 789 | ShapedType::kDynamic); |
| 790 | DenseI64ArrayAttr newConstOffsetsAttr = |
| 791 | rewriter.getDenseI64ArrayAttr(values: newConstOffsets); |
| 792 | |
| 793 | auto newOp = xegpu::LoadMatrixOp::create( |
| 794 | builder&: rewriter, location: loc, res: *distPayloadTyOrFailure, mem_desc: adaptor.getMemDesc(), |
| 795 | offsets: ValueRange(newCoords), const_offsets: newConstOffsetsAttr, subgroup_block_io: op.getSubgroupBlockIoAttr(), |
| 796 | layout: xegpu::DistributeLayoutAttr{}); |
| 797 | rewriter.replaceOp(op, newValues: newOp.getResult()); |
| 798 | return success(); |
| 799 | } |
| 800 | }; |
| 801 | |
| 802 | /// Distributes a subgroup-level vector.transpose op to lane-level. |
| 803 | struct SgToLaneVectorTranspose |
| 804 | : public OpConversionPattern<vector::TransposeOp> { |
| 805 | using OpConversionPattern<vector::TransposeOp>::OpConversionPattern; |
| 806 | |
| 807 | LogicalResult |
| 808 | matchAndRewrite(vector::TransposeOp op, OpAdaptor adaptor, |
| 809 | ConversionPatternRewriter &rewriter) const override { |
| 810 | xegpu::DistributeLayoutAttr sourceLayout = |
| 811 | xegpu::getTemporaryLayout(operandOrResult: op->getOpOperand(idx: 0)); |
| 812 | xegpu::DistributeLayoutAttr resultLayout = |
| 813 | xegpu::getTemporaryLayout(operandOrResult: op->getOpResult(idx: 0)); |
| 814 | if (!sourceLayout || !resultLayout) |
| 815 | return rewriter.notifyMatchFailure( |
| 816 | arg&: op, msg: "the source or result vector of the transpose op lacks layout " |
| 817 | "attribute" ); |
| 818 | ArrayRef<int64_t> perm = op.getPermutation(); |
| 819 | // Result layout must be a transpose of source layout. |
| 820 | if (!resultLayout.isTransposeOf(other: sourceLayout, perm, |
| 821 | kind: xegpu::LayoutKind::Lane)) |
| 822 | return rewriter.notifyMatchFailure( |
| 823 | arg&: op, msg: "the source or result vector layouts must be transposes of " |
| 824 | "each other" ); |
| 825 | FailureOr<VectorType> distributedResultTypeOrFailure = |
| 826 | getDistVecTypeBasedOnLaneLayout(layout: resultLayout, originalType: op.getResultVectorType()); |
| 827 | if (failed(Result: distributedResultTypeOrFailure)) |
| 828 | return rewriter.notifyMatchFailure( |
| 829 | arg&: op, msg: "Failed to distribute the result vector type in " |
| 830 | "vector::Transpose op" ); |
| 831 | auto newOp = vector::TransposeOp::create(builder&: rewriter, location: op.getLoc(), |
| 832 | vector: adaptor.getVector(), permutation: perm); |
| 833 | rewriter.replaceOp(op, newValues: castValueTo(rewriter, v: newOp.getResult(), |
| 834 | expectedTy: distributedResultTypeOrFailure.value())); |
| 835 | return success(); |
| 836 | } |
| 837 | }; |
| 838 | |
| 839 | /// Distributes a subgroup-level vector.bitcast op to lane-level. |
| 840 | /// Bitcast only impacts the innermost dimension of the source/result vectors. |
| 841 | struct SgToLaneVectorBitcast : public OpConversionPattern<vector::BitCastOp> { |
| 842 | using OpConversionPattern<vector::BitCastOp>::OpConversionPattern; |
| 843 | |
| 844 | LogicalResult |
| 845 | matchAndRewrite(vector::BitCastOp op, OpAdaptor adaptor, |
| 846 | ConversionPatternRewriter &rewriter) const override { |
| 847 | xegpu::DistributeLayoutAttr resultLayout = |
| 848 | xegpu::getTemporaryLayout(operandOrResult: op->getOpResult(idx: 0)); |
| 849 | if (!resultLayout) |
| 850 | return rewriter.notifyMatchFailure( |
| 851 | arg&: op, msg: "result vector of the bitcast op lacks layout attribute" ); |
| 852 | FailureOr<VectorType> distributedResultTypeOrFailure = |
| 853 | getDistVecTypeBasedOnLaneLayout(layout: resultLayout, originalType: op.getResultVectorType()); |
| 854 | if (failed(Result: distributedResultTypeOrFailure)) |
| 855 | return rewriter.notifyMatchFailure( |
| 856 | arg&: op, msg: "Failed to distribute the result vector type in " |
| 857 | "vector::BitCast op" ); |
| 858 | auto newOp = vector::BitCastOp::create( |
| 859 | builder&: rewriter, location: op.getLoc(), result: distributedResultTypeOrFailure.value(), |
| 860 | source: adaptor.getSource()); |
| 861 | rewriter.replaceOp(op, newValues: newOp.getResult()); |
| 862 | return success(); |
| 863 | } |
| 864 | }; |
| 865 | |
| 866 | /// Distributes a subgroup-level vector.create_mask or vector.constant_mask op |
| 867 | /// to lane-level. |
| 868 | /// The pattern constructs a mask based on the following bounds check: |
| 869 | /// ``` |
| 870 | /// for d in [0, ..., maskRank): |
| 871 | /// mask &= (staticOffset[d] < (bound[d] - base[d])) |
| 872 | /// ``` |
| 873 | /// where |
| 874 | /// - `base` is the coordinate vector of the first distributed unit. |
| 875 | /// - `staticOffset` is the offsets vector per element. |
| 876 | /// - `bound` is the original mask bound for the corresponding dimension. |
| 877 | /// The mask vector contains *all* elements (i.e., non-unit `lane_data` is |
| 878 | /// flattened). For example, |
| 879 | /// ``` |
| 880 | /// %mask = vector.create_mask %bound_0, %bound_1 { |
| 881 | /// lane_layout = [1, 16], lane_data = [2, 1] |
| 882 | /// }: vector<8x32xi1> |
| 883 | /// ``` |
| 884 | /// Has 8 dist units (of shape [2, 1]) with offsets for lane 0: |
| 885 | /// { |
| 886 | /// [0, 0], [0, 16], |
| 887 | /// [2, 0], [2, 16], |
| 888 | /// [4, 0], [4, 16], |
| 889 | /// [6, 0], [6, 16] |
| 890 | /// } |
| 891 | /// We check the distance to the bound from the first dist. unit: |
| 892 | /// %base_0 = genCoords(gpu.lane_id, layout)[0][0] |
| 893 | /// %base_1 = genCoords(gpu.lane_id, layout)[0][1] |
| 894 | /// %baseDistanceToBound_0 = %bound_0 - %base_0 |
| 895 | /// %baseDistanceToBound_1 = %bound_1 - %base_1 |
| 896 | /// The pattern flattens the dimension d coordinates of each element: |
| 897 | /// %staticOffset_0 = [0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7] |
| 898 | /// %staticOffset_1 = [0, 16, 0, 16, 0, 16, 0, 16, 0, 16, 0, 16, 0, 16, 0, 16] |
| 899 | /// and compares each coordinate against the corresponding mask bound dim: |
| 900 | /// %mask_0 = arith.cmpi slt, %staticOffset_0, bcast(%baseDistanceToBound_0) |
| 901 | /// %mask_1 = arith.cmpi slt, %staticOffset_1, bcast(%baseDistanceToBound_1) |
| 902 | /// %mask = shape_cast(%mask_0 & %mask_1) : vector<16xi1> to vector<8x2xi1> |
| 903 | /// |
| 904 | template <typename OpType, |
| 905 | typename = std::enable_if_t<llvm::is_one_of< |
| 906 | OpType, vector::CreateMaskOp, vector::ConstantMaskOp>::value>> |
| 907 | struct SgToLaneCreateMask : public OpConversionPattern<OpType> { |
| 908 | using OpConversionPattern<OpType>::OpConversionPattern; |
| 909 | |
| 910 | LogicalResult |
| 911 | matchAndRewrite(OpType op, typename OpType::Adaptor adaptor, |
| 912 | ConversionPatternRewriter &rewriter) const override { |
| 913 | xegpu::DistributeLayoutAttr layout = |
| 914 | xegpu::getTemporaryLayout(op->getOpResult(0)); |
| 915 | if (!layout || !layout.isForSubgroup()) |
| 916 | return rewriter.notifyMatchFailure( |
| 917 | op, "operation result does not have subgroup distribute layout" ); |
| 918 | |
| 919 | VectorType origType = op.getType(); |
| 920 | FailureOr<VectorType> distTypeOrFailure = |
| 921 | getDistVecTypeBasedOnLaneLayout(layout, originalType: origType); |
| 922 | if (failed(Result: distTypeOrFailure)) |
| 923 | return rewriter.notifyMatchFailure( |
| 924 | op, "unable to compute lane vector type from the layout" ); |
| 925 | |
| 926 | VectorType distType = distTypeOrFailure.value(); |
| 927 | Location loc = op.getLoc(); |
| 928 | |
| 929 | // Materialize the original mask bounds as Values. |
| 930 | SmallVector<Value> origBounds; |
| 931 | if constexpr (std::is_same_v<OpType, vector::CreateMaskOp>) { |
| 932 | origBounds.append(op.getOperands().begin(), op.getOperands().end()); |
| 933 | } else { |
| 934 | auto dimSizes = op.getMaskDimSizesAttr().asArrayRef(); |
| 935 | for (auto dimSize : dimSizes) |
| 936 | origBounds.push_back( |
| 937 | Elt: arith::ConstantIndexOp::create(rewriter, loc, dimSize).getResult()); |
| 938 | } |
| 939 | |
| 940 | ArrayRef<int64_t> origShape = origType.getShape(); |
| 941 | |
| 942 | // Use computeDistributedCoords to get the coordinates each WI owns. |
| 943 | Value laneId = gpu::LaneIdOp::create(builder&: rewriter, location: loc, result: rewriter.getIndexType(), |
| 944 | /*upperBound=*/upper_bound: mlir::IntegerAttr()); |
| 945 | auto maybeCoordsVec = |
| 946 | layout.computeDistributedCoords(builder&: rewriter, loc, linearId: laneId, shape: origShape); |
| 947 | if (failed(Result: maybeCoordsVec)) |
| 948 | return rewriter.notifyMatchFailure( |
| 949 | op, "failed to compute distributed coordinates from layout" ); |
| 950 | |
| 951 | SmallVector<SmallVector<Value>> laneDataCoords = maybeCoordsVec.value(); |
| 952 | SmallVector<int64_t> laneData = layout.getEffectiveLaneDataAsInt(); |
| 953 | ArrayRef<int64_t> distShape = distType.getShape(); |
| 954 | int64_t rank = distType.getRank(); |
| 955 | int64_t numElements = distType.getNumElements(); |
| 956 | |
| 957 | if (static_cast<int64_t>(laneData.size()) != rank || |
| 958 | !computeShapeRatio(shape: distShape, subShape: laneData)) |
| 959 | return rewriter.notifyMatchFailure( |
| 960 | op, "lane_data does not tile the distributed vector" ); |
| 961 | |
| 962 | SmallVector<int64_t> distStrides = computeStrides(sizes: distShape); |
| 963 | assert(static_cast<int64_t>(laneDataCoords.size()) * |
| 964 | computeProduct(laneData) == |
| 965 | numElements && |
| 966 | "number of coordinate sets must match number of lane_data blocks" ); |
| 967 | |
| 968 | // Static offsets to be applied to a dynamic lane's dist units coordinates. |
| 969 | SmallVector<SmallVector<int64_t>> staticLaneDataOffsetsOrig = |
| 970 | layout.computeStaticDistributedCoords(/*linearId=*/0, shape: origShape); |
| 971 | if (staticLaneDataOffsetsOrig.empty() || |
| 972 | staticLaneDataOffsetsOrig.size() != laneDataCoords.size()) |
| 973 | return rewriter.notifyMatchFailure( |
| 974 | op, "static and dynamic coordinates disagree on the distribution " |
| 975 | "unit count" ); |
| 976 | |
| 977 | SmallVector<int64_t> unitTile(rank, 1); |
| 978 | int64_t linearLaneDataIdx = 0; |
| 979 | // Layout and shape information is static, compute static offset of lane's |
| 980 | // elements in the source dimensions. Each dim is a flat vec of element |
| 981 | // offsets. We store dim-major to have an easy materialization as one const |
| 982 | // vector. |
| 983 | SmallVector<SmallVector<int64_t>> staticElemOffset( |
| 984 | rank, SmallVector<int64_t>(numElements)); |
| 985 | // For each lane_data block in the lane's dist shape |
| 986 | for (SmallVector<int64_t> laneDataOffsetDist : |
| 987 | StaticTileOffsetRange(distShape, laneData)) { |
| 988 | ArrayRef<int64_t> staticLaneDataOffsetOrig = |
| 989 | staticLaneDataOffsetsOrig[linearLaneDataIdx++]; |
| 990 | // For each element in the lane_data block |
| 991 | for (SmallVector<int64_t> elemOffsetInLaneData : |
| 992 | StaticTileOffsetRange(laneData, unitTile)) { |
| 993 | // For each dim of indexing space |
| 994 | SmallVector<int64_t> elementOffsetDist(rank); |
| 995 | for (int64_t d = 0; d < rank; d++) |
| 996 | elementOffsetDist[d] = |
| 997 | laneDataOffsetDist[d] + elemOffsetInLaneData[d]; |
| 998 | int64_t elemLinearizedIdxDist = |
| 999 | linearize(offsets: elementOffsetDist, basis: distStrides); |
| 1000 | for (int64_t d = 0; d < rank; d++) |
| 1001 | staticElemOffset[d][elemLinearizedIdxDist] = |
| 1002 | staticLaneDataOffsetOrig[d] + elemOffsetInLaneData[d]; |
| 1003 | } |
| 1004 | } |
| 1005 | |
| 1006 | // Check, whether ALL elements of the distributed mask are within the valid |
| 1007 | // extent of EACH source dimension. |
| 1008 | // Expressed as dyn_offset[d] + static_offset[d] < dyn_mask_bound[d], |
| 1009 | // or equivalently, |
| 1010 | // static_offset[d] < dyn_mask_bound[d] - dyn_offset[d] |
| 1011 | auto flatIndexType = VectorType::get(shape: numElements, elementType: rewriter.getIndexType()); |
| 1012 | Value inBounds; |
| 1013 | for (int64_t d = 0; d < rank; d++) { |
| 1014 | std::optional<int64_t> constBound = getConstantIntValue(ofr: origBounds[d]); |
| 1015 | if (constBound && *constBound >= origShape[d]) |
| 1016 | continue; |
| 1017 | |
| 1018 | Value materializedStaticOffset = arith::ConstantOp::create( |
| 1019 | builder&: rewriter, location: loc, value: rewriter.getIndexVectorAttr(values: staticElemOffset[d])); |
| 1020 | // Consider only the first unit, all others are compile-time multiple |
| 1021 | // offsets of the first one and are already encoded in staticOffsets. |
| 1022 | Value validDimExtent = arith::SubIOp::create(builder&: rewriter, location: loc, lhs: origBounds[d], |
| 1023 | rhs: laneDataCoords[0][d]); |
| 1024 | // Dim extent applies to all elements. |
| 1025 | Value validDimExtentPerElement = vector::BroadcastOp::create( |
| 1026 | builder&: rewriter, location: loc, result: flatIndexType, source: validDimExtent); |
| 1027 | Value elementMaskInDim = arith::CmpIOp::create( |
| 1028 | builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::slt, lhs: materializedStaticOffset, |
| 1029 | rhs: validDimExtentPerElement); |
| 1030 | // An element is valid iff it is within the valid extent of ALL |
| 1031 | // dimensions. |
| 1032 | inBounds = inBounds ? arith::AndIOp::create(builder&: rewriter, location: loc, lhs: inBounds, |
| 1033 | rhs: elementMaskInDim) |
| 1034 | .getResult() |
| 1035 | : elementMaskInDim; |
| 1036 | } |
| 1037 | // Every dim's bound saturates the mask extent, so no comparison was |
| 1038 | // emitted at all: every element of every lane is in bounds. |
| 1039 | if (!inBounds) { |
| 1040 | rewriter.replaceOp(op, arith::ConstantOp::create( |
| 1041 | builder&: rewriter, location: loc, result: distType, |
| 1042 | value: DenseElementsAttr::get(type: distType, value: true))); |
| 1043 | return success(); |
| 1044 | } |
| 1045 | auto resMask = |
| 1046 | rewriter.createOrFold<vector::ShapeCastOp>(location: loc, args&: distType, args&: inBounds); |
| 1047 | rewriter.replaceOp(op, resMask); |
| 1048 | return success(); |
| 1049 | } |
| 1050 | }; |
| 1051 | |
| 1052 | /// This pattern distributes a subgroup-level StoreMatrix op to lane-level. |
| 1053 | struct SgToLaneStoreMatrix : public OpConversionPattern<xegpu::StoreMatrixOp> { |
| 1054 | using OpConversionPattern<xegpu::StoreMatrixOp>::OpConversionPattern; |
| 1055 | |
| 1056 | LogicalResult |
| 1057 | matchAndRewrite(xegpu::StoreMatrixOp op, OpAdaptor adaptor, |
| 1058 | ConversionPatternRewriter &rewriter) const override { |
| 1059 | auto layout = op.getLayoutAttr(); |
| 1060 | // If no layout, nothing to do. |
| 1061 | if (!layout) |
| 1062 | return failure(); |
| 1063 | |
| 1064 | VectorType sgPayloadTy = dyn_cast<VectorType>(Val: op.getData().getType()); |
| 1065 | if (!sgPayloadTy) |
| 1066 | return rewriter.notifyMatchFailure( |
| 1067 | arg&: op, msg: "the matrix op payload must be a vector type" ); |
| 1068 | |
| 1069 | auto loc = op.getLoc(); |
| 1070 | auto offsets = op.getMixedOffsets(); |
| 1071 | if (offsets.empty()) |
| 1072 | return rewriter.notifyMatchFailure(arg&: op, msg: "the store op must have offsets" ); |
| 1073 | |
| 1074 | FailureOr<VectorType> distPayloadTyOrFailure = |
| 1075 | getDistVecTypeBasedOnLaneLayout(layout, originalType: sgPayloadTy); |
| 1076 | if (failed(Result: distPayloadTyOrFailure)) |
| 1077 | return rewriter.notifyMatchFailure( |
| 1078 | arg&: op, msg: "Failed to distribute matrix op payload based on layout." ); |
| 1079 | |
| 1080 | SmallVector<Value> offsetsAsValues = |
| 1081 | vector::getAsValues(builder&: rewriter, loc, foldResults: offsets); |
| 1082 | |
| 1083 | SmallVector<Value> newCoords = offsetsAsValues; |
| 1084 | if (!op.getSubgroupBlockIoAttr()) { |
| 1085 | newCoords = computeDistributedCoordsForMatrixOp( |
| 1086 | rewriter, loc, layout, payloadShape: sgPayloadTy.getShape(), origOffsets: offsetsAsValues); |
| 1087 | if (newCoords.empty()) |
| 1088 | return rewriter.notifyMatchFailure( |
| 1089 | arg&: op, msg: "Failed to compute distributed coordinates." ); |
| 1090 | } |
| 1091 | |
| 1092 | SmallVector<int64_t> newConstOffsets(op.getConstOffsets().size(), |
| 1093 | ShapedType::kDynamic); |
| 1094 | DenseI64ArrayAttr newConstOffsetsAttr = |
| 1095 | rewriter.getDenseI64ArrayAttr(values: newConstOffsets); |
| 1096 | |
| 1097 | xegpu::StoreMatrixOp::create( |
| 1098 | builder&: rewriter, location: loc, resultTypes: TypeRange{}, |
| 1099 | data: castValueTo(rewriter, v: cast<TypedValue<VectorType>>(Val: adaptor.getData()), |
| 1100 | expectedTy: distPayloadTyOrFailure.value()), |
| 1101 | mem_desc: adaptor.getMemDesc(), offsets: ValueRange(newCoords), const_offsets: newConstOffsetsAttr, |
| 1102 | subgroup_block_io: op.getSubgroupBlockIoAttr(), layout: xegpu::DistributeLayoutAttr{}); |
| 1103 | rewriter.eraseOp(op); |
| 1104 | return success(); |
| 1105 | } |
| 1106 | }; |
| 1107 | |
| 1108 | /// Distributes a subgroup-level StoreScatter (xegpu.store) op to |
| 1109 | /// lane-level. |
| 1110 | /// |
| 1111 | /// Example 1 (1D): |
| 1112 | /// layout = #xegpu.layout<lane_layout = [16], lane_data = [1]> |
| 1113 | /// %mask = producer_op : vector<16xi1> |
| 1114 | /// %offset = producer_op : vector<16xindex> |
| 1115 | /// xegpu.store %payload, %src[%offset], %mask : vector<16xf16>, |
| 1116 | /// memref<256xf16>, vector<16xindex>, vector<16xi1> |
| 1117 | /// Distributed to: |
| 1118 | /// %mask = producer_op : vector<1xi1> |
| 1119 | /// %offset = producer_op : vector<1xindex> |
| 1120 | /// xegpu.store %payload, %src[%offset], %mask : vector<1xf16>, |
| 1121 | /// memref<256xf16>, vector<1xindex>, vector<1xi1> |
| 1122 | /// |
| 1123 | /// Example 2 (3D with leading unit dims): |
| 1124 | /// layout = #xegpu.layout<lane_layout = [1, 1, 16], lane_data = [1, 1, 1]> |
| 1125 | /// %mask = producer_op : vector<1x1x16xi1> |
| 1126 | /// %offset = producer_op : vector<1x1x16xindex> |
| 1127 | /// xegpu.store %payload, %src[%offset], %mask : vector<1x1x16xf16>, |
| 1128 | /// memref<256xf16>, vector<1x1x16xindex>, vector<1x1x16xi1> |
| 1129 | /// Distributed to: |
| 1130 | /// %mask = producer_op : vector<1x1x1xi1> |
| 1131 | /// %offset = producer_op : vector<1x1x1xindex> |
| 1132 | /// xegpu.store %payload, %src[%offset], %mask : vector<1xf16>, |
| 1133 | /// memref<256xf16>, vector<1xindex>, vector<1xi1> |
| 1134 | struct SgToLaneStoreScatter |
| 1135 | : public OpConversionPattern<xegpu::StoreScatterOp> { |
| 1136 | using OpConversionPattern<xegpu::StoreScatterOp>::OpConversionPattern; |
| 1137 | |
| 1138 | LogicalResult |
| 1139 | matchAndRewrite(xegpu::StoreScatterOp op, OpAdaptor adaptor, |
| 1140 | ConversionPatternRewriter &rewriter) const override { |
| 1141 | xegpu::DistributeLayoutAttr layout = op.getAnchorLayout(); |
| 1142 | if (!layout) |
| 1143 | return failure(); |
| 1144 | |
| 1145 | VectorType origValueTy = op.getValueType(); |
| 1146 | if (!origValueTy) |
| 1147 | return failure(); |
| 1148 | |
| 1149 | // Check that all leading dimensions are unit dimensions. |
| 1150 | int effectiveVecRank = 1; |
| 1151 | ArrayRef<int64_t> shape = origValueTy.getShape(); |
| 1152 | if (llvm::any_of(Range: shape.take_front(N: origValueTy.getRank() - effectiveVecRank), |
| 1153 | P: [](int64_t d) { return d != 1; })) |
| 1154 | return rewriter.notifyMatchFailure( |
| 1155 | arg&: op, msg: "Only unit dimensions allowed for the leading " |
| 1156 | "dimensions of the store vector!" ); |
| 1157 | |
| 1158 | auto distValueTyOrFailure = |
| 1159 | xegpu::getDistVecTypeBasedOnLaneLayout(layout, originalType: origValueTy); |
| 1160 | if (failed(Result: distValueTyOrFailure)) |
| 1161 | return rewriter.notifyMatchFailure( |
| 1162 | arg&: op, msg: "unable to compute expected lane vector type from lane layout" ); |
| 1163 | |
| 1164 | VectorType distValueTy = distValueTyOrFailure.value(); |
| 1165 | VectorType distValueTy1D = VectorType::get(shape: {distValueTy.getNumElements()}, |
| 1166 | elementType: distValueTy.getElementType()); |
| 1167 | |
| 1168 | Value distValue = adaptor.getValue(); |
| 1169 | if (distValue.getType() != distValueTy1D) |
| 1170 | distValue = castValueTo(rewriter, v: cast<TypedValue<VectorType>>(Val&: distValue), |
| 1171 | expectedTy: distValueTy1D); |
| 1172 | |
| 1173 | // Flatten offsets and mask to 1D to match the 1D value type. |
| 1174 | Value distOffsets = adaptor.getOffsets(); |
| 1175 | auto distOffsetsTy = cast<VectorType>(Val: distOffsets.getType()); |
| 1176 | VectorType offsetsTy1D = VectorType::get(shape: {distOffsetsTy.getNumElements()}, |
| 1177 | elementType: distOffsetsTy.getElementType()); |
| 1178 | distOffsets = castValueTo( |
| 1179 | rewriter, v: cast<TypedValue<VectorType>>(Val&: distOffsets), expectedTy: offsetsTy1D); |
| 1180 | |
| 1181 | Value distMask = adaptor.getMask(); |
| 1182 | auto distMaskTy = cast<VectorType>(Val: distMask.getType()); |
| 1183 | VectorType maskTy1D = VectorType::get(shape: {distMaskTy.getNumElements()}, |
| 1184 | elementType: distMaskTy.getElementType()); |
| 1185 | distMask = |
| 1186 | castValueTo(rewriter, v: cast<TypedValue<VectorType>>(Val&: distMask), expectedTy: maskTy1D); |
| 1187 | |
| 1188 | Value distDest = adaptor.getDest(); |
| 1189 | xegpu::StoreScatterOp::create(builder&: rewriter, location: op.getLoc(), value: distValue, dest: distDest, |
| 1190 | offsets: distOffsets, mask: distMask, l1_hint: op.getL1HintAttr(), |
| 1191 | l2_hint: op.getL2HintAttr(), l3_hint: op.getL3HintAttr(), |
| 1192 | /*layout=*/nullptr, |
| 1193 | /*contiguity=*/nullptr); |
| 1194 | rewriter.eraseOp(op); |
| 1195 | return success(); |
| 1196 | } |
| 1197 | }; |
| 1198 | |
| 1199 | /// Distribute a vector::StepOp to lane-level. |
| 1200 | /// The layout must have exactly 1 effective lane dimension. |
| 1201 | /// We completely resolve the vector::StepOp by computing the lane_data-sized |
| 1202 | /// subranges. |
| 1203 | struct SgToLaneVectorStep : public OpConversionPattern<vector::StepOp> { |
| 1204 | using OpConversionPattern<vector::StepOp>::OpConversionPattern; |
| 1205 | |
| 1206 | LogicalResult |
| 1207 | matchAndRewrite(vector::StepOp op, OpAdaptor adaptor, |
| 1208 | ConversionPatternRewriter &rewriter) const override { |
| 1209 | xegpu::DistributeLayoutAttr resultLayout = |
| 1210 | xegpu::getTemporaryLayout(operandOrResult: op->getResult(idx: 0)); |
| 1211 | if (!resultLayout || !resultLayout.isForSubgroup()) |
| 1212 | return rewriter.notifyMatchFailure( |
| 1213 | arg&: op, msg: "the result vector of the step op lacks subgroup layout" ); |
| 1214 | |
| 1215 | auto loc = op.getLoc(); |
| 1216 | auto stepResultVecTy = op.getResult().getType(); |
| 1217 | auto laneShapeOrFailure = |
| 1218 | xegpu::getDistVecTypeBasedOnLaneLayout(layout: resultLayout, originalType: stepResultVecTy); |
| 1219 | if (failed(Result: laneShapeOrFailure)) |
| 1220 | return rewriter.notifyMatchFailure( |
| 1221 | arg&: op, msg: "unable to compute lane vector type from the layout" ); |
| 1222 | VectorType newVecTy = laneShapeOrFailure.value(); |
| 1223 | |
| 1224 | Value laneId = gpu::LaneIdOp::create(builder&: rewriter, location: loc, result: rewriter.getIndexType(), |
| 1225 | /*upperBound=*/upper_bound: mlir::IntegerAttr()); |
| 1226 | auto laneDataBlockCoords = resultLayout.computeDistributedCoords( |
| 1227 | builder&: rewriter, loc, linearId: laneId, shape: stepResultVecTy.getShape()); |
| 1228 | if (failed(Result: laneDataBlockCoords)) |
| 1229 | return rewriter.notifyMatchFailure( |
| 1230 | arg&: op, msg: "failed to compute lane data block coordinates" ); |
| 1231 | |
| 1232 | auto laneDataBlockCoordsVec = laneDataBlockCoords.value(); |
| 1233 | auto laneDataBlockLength = resultLayout.getEffectiveLaneDataAsInt()[0]; |
| 1234 | assert(static_cast<int64_t>(laneDataBlockCoordsVec.size()) == |
| 1235 | newVecTy.getNumElements() / laneDataBlockLength); |
| 1236 | SmallVector<Value> stepVals; |
| 1237 | // For each lane_data block, reconstruct its sub-range |
| 1238 | // from the range of SG-level vector.step.Example: vector.step |
| 1239 | // {slice<layout<lane_layout=[2,4,2], lane_data=[1,2,1]>, dims=[0,2]>} : |
| 1240 | // vector<16xindex> |
| 1241 | // Each logical lane holds 4 elements as 2 blocks of 2 elements each. |
| 1242 | // The blocks are round-robin distributed, so logical lane id 0 |
| 1243 | // holds values [0,1, 8,9]. |
| 1244 | for (auto &laneDataBlockCoords : laneDataBlockCoordsVec) { |
| 1245 | auto laneDataBlockStartCoord = laneDataBlockCoords[0]; |
| 1246 | stepVals.push_back(Elt: laneDataBlockStartCoord); |
| 1247 | for (int i = 1; i < laneDataBlockLength; ++i) { |
| 1248 | auto offset = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: i); |
| 1249 | stepVals.push_back(Elt: arith::AddIOp::create( |
| 1250 | builder&: rewriter, location: loc, lhs: laneDataBlockStartCoord, rhs: offset)); |
| 1251 | } |
| 1252 | } |
| 1253 | assert(static_cast<int64_t>(stepVals.size()) == newVecTy.getNumElements() && |
| 1254 | "Expecting the number of step values to match the number of " |
| 1255 | "elements in the vector" ); |
| 1256 | auto stepOpVal = |
| 1257 | vector::FromElementsOp::create(builder&: rewriter, location: loc, result: newVecTy, elements: stepVals); |
| 1258 | rewriter.replaceOp(op, newOp: stepOpVal); |
| 1259 | return success(); |
| 1260 | } |
| 1261 | }; |
| 1262 | |
| 1263 | /// Distributes a subgroup-level vector.extract op to lane-level. Only |
| 1264 | /// handles sub-vector extraction (result is VectorType, not scalar). |
| 1265 | struct : public OpConversionPattern<vector::ExtractOp> { |
| 1266 | using OpConversionPattern<vector::ExtractOp>::OpConversionPattern; |
| 1267 | |
| 1268 | LogicalResult |
| 1269 | matchAndRewrite(vector::ExtractOp op, OpAdaptor adaptor, |
| 1270 | ConversionPatternRewriter &rewriter) const override { |
| 1271 | // Only handle vector results (not scalar extraction). |
| 1272 | auto resultType = dyn_cast<VectorType>(Val: op.getType()); |
| 1273 | if (!resultType) |
| 1274 | return rewriter.notifyMatchFailure(arg&: op, msg: "scalar extract not supported" ); |
| 1275 | |
| 1276 | xegpu::DistributeLayoutAttr layout = |
| 1277 | xegpu::getTemporaryLayout(operandOrResult: op->getOpResult(idx: 0)); |
| 1278 | if (!layout || !layout.isForSubgroup()) |
| 1279 | return failure(); |
| 1280 | |
| 1281 | // This implementation assumes distribution only happens on the innermost |
| 1282 | // dimension. Verify that lane_layout[0...n-2] are all unit. |
| 1283 | auto laneLayout = layout.getEffectiveLaneLayoutAsInt(); |
| 1284 | if (llvm::any_of(Range: ArrayRef<int64_t>(laneLayout).drop_back(N: 1), |
| 1285 | P: [](int64_t v) { return v != 1; })) |
| 1286 | return rewriter.notifyMatchFailure( |
| 1287 | arg&: op, msg: "only innermost dimension distribution is supported for " |
| 1288 | "vector.extract" ); |
| 1289 | |
| 1290 | auto newOp = vector::ExtractOp::create( |
| 1291 | builder&: rewriter, location: op.getLoc(), source: adaptor.getSource(), position: op.getMixedPosition()); |
| 1292 | rewriter.replaceOp(op, newValues: newOp.getResult()); |
| 1293 | return success(); |
| 1294 | } |
| 1295 | }; |
| 1296 | |
| 1297 | /// This pattern distributes a subgroup-level ShapeCast op to lane-level. |
| 1298 | struct SgToLaneVectorShapeCast |
| 1299 | : public OpConversionPattern<vector::ShapeCastOp> { |
| 1300 | using OpConversionPattern<vector::ShapeCastOp>::OpConversionPattern; |
| 1301 | |
| 1302 | LogicalResult |
| 1303 | matchAndRewrite(vector::ShapeCastOp op, OpAdaptor adaptor, |
| 1304 | ConversionPatternRewriter &rewriter) const override { |
| 1305 | xegpu::DistributeLayoutAttr resultLayout = |
| 1306 | xegpu::getTemporaryLayout(operandOrResult: op->getOpResult(idx: 0)); |
| 1307 | if (!resultLayout || !resultLayout.isForSubgroup()) |
| 1308 | return rewriter.notifyMatchFailure( |
| 1309 | arg&: op, msg: "the result vector of the shape_cast op lacks subgroup layout" ); |
| 1310 | |
| 1311 | auto resultDistTypeOrFailure = xegpu::getDistVecTypeBasedOnLaneLayout( |
| 1312 | layout: resultLayout, originalType: op.getResultVectorType()); |
| 1313 | if (failed(Result: resultDistTypeOrFailure)) |
| 1314 | return rewriter.notifyMatchFailure( |
| 1315 | arg&: op, msg: "failed to get distributed vector type for result" ); |
| 1316 | |
| 1317 | Value source = adaptor.getSource(); |
| 1318 | auto newShapeCast = vector::ShapeCastOp::create( |
| 1319 | builder&: rewriter, location: op.getLoc(), result: resultDistTypeOrFailure.value(), source); |
| 1320 | rewriter.replaceOp(op, newOp: newShapeCast); |
| 1321 | return success(); |
| 1322 | } |
| 1323 | }; |
| 1324 | |
| 1325 | /// Distributes a subgroup-level vector.extract_strided_slice op to |
| 1326 | /// lane-level. If the result is distributed, the offsets and sizes are |
| 1327 | /// adjusted to match the distributed types. |
| 1328 | struct |
| 1329 | : public OpConversionPattern<vector::ExtractStridedSliceOp> { |
| 1330 | using OpConversionPattern<vector::ExtractStridedSliceOp>::OpConversionPattern; |
| 1331 | |
| 1332 | LogicalResult |
| 1333 | matchAndRewrite(vector::ExtractStridedSliceOp op, OpAdaptor adaptor, |
| 1334 | ConversionPatternRewriter &rewriter) const override { |
| 1335 | xegpu::DistributeLayoutAttr resultLayout = |
| 1336 | xegpu::getTemporaryLayout(operandOrResult: op->getOpResult(idx: 0)); |
| 1337 | if (!resultLayout || !resultLayout.isForSubgroup()) |
| 1338 | return failure(); |
| 1339 | |
| 1340 | VectorType resultType = op.getType(); |
| 1341 | auto distResultTyOrFailure = |
| 1342 | xegpu::getDistVecTypeBasedOnLaneLayout(layout: resultLayout, originalType: resultType); |
| 1343 | if (failed(Result: distResultTyOrFailure)) |
| 1344 | return rewriter.notifyMatchFailure( |
| 1345 | arg&: op, msg: "unable to compute distributed vector type from lane layout" ); |
| 1346 | VectorType distResultTy = *distResultTyOrFailure; |
| 1347 | |
| 1348 | SmallVector<int64_t> distributedDims = |
| 1349 | getDistributedDims(originalType: resultType, distributedType: distResultTy); |
| 1350 | |
| 1351 | // Collect updated sizes, offsets, strides. Pad to full source rank. |
| 1352 | int64_t sourceRank = op.getSourceVectorType().getRank(); |
| 1353 | SmallVector<Attribute> updatedSizes = |
| 1354 | llvm::map_to_vector(C: op.getSizes(), F: [](Attribute attr) { return attr; }); |
| 1355 | SmallVector<Attribute> updatedOffsets = llvm::map_to_vector( |
| 1356 | C: op.getOffsets(), F: [](Attribute attr) { return attr; }); |
| 1357 | SmallVector<Attribute> updatedStrides = llvm::map_to_vector( |
| 1358 | C: op.getStrides(), F: [](Attribute attr) { return attr; }); |
| 1359 | for (int64_t i = op.getSizes().size(); i < sourceRank; ++i) { |
| 1360 | updatedSizes.push_back( |
| 1361 | Elt: rewriter.getI64IntegerAttr(value: op.getSourceVectorType().getDimSize(idx: i))); |
| 1362 | updatedOffsets.push_back(Elt: rewriter.getI64IntegerAttr(value: 0)); |
| 1363 | updatedStrides.push_back(Elt: rewriter.getI64IntegerAttr(value: 1)); |
| 1364 | } |
| 1365 | |
| 1366 | // Each distributed dim shrinks by its own lane count, so its size and |
| 1367 | // offset are rescaled by that count. |
| 1368 | if (!distributedDims.empty()) { |
| 1369 | auto sourceLayout = xegpu::getTemporaryLayout(operandOrResult: op->getOpOperand(idx: 0)); |
| 1370 | if (!sourceLayout || sourceLayout.getEffectiveLaneLayoutAsInt().empty()) |
| 1371 | return rewriter.notifyMatchFailure( |
| 1372 | arg&: op, msg: "source of extract_strided_slice lacks distribution layout" ); |
| 1373 | SmallVector<int64_t> laneLayout = |
| 1374 | sourceLayout.getEffectiveLaneLayoutAsInt(); |
| 1375 | SmallVector<int64_t> laneData = sourceLayout.getEffectiveLaneDataAsInt(); |
| 1376 | ArrayRef<int64_t> sourceShape = op.getSourceVectorType().getShape(); |
| 1377 | for (int64_t distDim : distributedDims) { |
| 1378 | int64_t lanes = laneLayout[distDim]; |
| 1379 | if (lanes == 0 || sourceShape[distDim] % lanes != 0) |
| 1380 | return rewriter.notifyMatchFailure( |
| 1381 | arg&: op, msg: "source size along a distributed dim is not a multiple of " |
| 1382 | "its lane count" ); |
| 1383 | int64_t distrDimOffset = |
| 1384 | cast<IntegerAttr>(Val&: updatedOffsets[distDim]).getInt(); |
| 1385 | if (distrDimOffset % (lanes * laneData[distDim]) != 0) |
| 1386 | return rewriter.notifyMatchFailure( |
| 1387 | arg&: op, msg: "offset along a distributed dim is not a multiple of its " |
| 1388 | "lane tile" ); |
| 1389 | updatedSizes[distDim] = |
| 1390 | rewriter.getI64IntegerAttr(value: distResultTy.getDimSize(idx: distDim)); |
| 1391 | updatedOffsets[distDim] = |
| 1392 | rewriter.getI64IntegerAttr(value: distrDimOffset / lanes); |
| 1393 | } |
| 1394 | } |
| 1395 | |
| 1396 | auto newOp = vector::ExtractStridedSliceOp::create( |
| 1397 | builder&: rewriter, location: op.getLoc(), resultType0: distResultTy, source: adaptor.getSource(), |
| 1398 | offsets: ArrayAttr::get(context: rewriter.getContext(), value: updatedOffsets), |
| 1399 | sizes: ArrayAttr::get(context: rewriter.getContext(), value: updatedSizes), |
| 1400 | strides: ArrayAttr::get(context: rewriter.getContext(), value: updatedStrides)); |
| 1401 | rewriter.replaceOp(op, newValues: newOp.getResult()); |
| 1402 | return success(); |
| 1403 | } |
| 1404 | }; |
| 1405 | |
| 1406 | /// This pattern distributes a subgroup-level `vector.broadcast` op to |
| 1407 | /// lane-level. The pattern supports three cases: |
| 1408 | /// |
| 1409 | /// 1) Broadcast a low-rank vector to high-rank vector: The low-rank input |
| 1410 | /// vector must have a slice layout of the result. If the distributed source |
| 1411 | /// and target vector types are identical, this lowers to a no-op; otherwise, |
| 1412 | /// it remains a broadcast but operates on distributed vectors. |
| 1413 | /// |
| 1414 | /// 2) Broadcast a same-rank vector with identical layouts for source and |
| 1415 | /// target: The source vector must have unit dimensions, and lane_data must |
| 1416 | /// be unit size for those unit dims. This always lowers to a no-op. |
| 1417 | /// |
| 1418 | /// 3) Broadcast a scalar with no layout: This always lowers to a broadcast |
| 1419 | /// from scalar to distributed result type. |
| 1420 | /// |
| 1421 | /// Example 1 (low-rank to high-rank broadcast): |
| 1422 | /// ``` |
| 1423 | /// %0 = "some_op"() {layout_result_0 = |
| 1424 | /// #xegpu.slice<#xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>, |
| 1425 | /// dims = [0]>} : () -> vector<16xf16> |
| 1426 | /// %1 = vector.broadcast %0 {layout_result_0 = |
| 1427 | /// #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>} |
| 1428 | /// : vector<16xf16> to vector<16x16xf16> |
| 1429 | /// ``` |
| 1430 | /// is distributed to: |
| 1431 | /// ``` |
| 1432 | /// %0 = "some_op"() : () -> vector<1xf16> |
| 1433 | /// %1 = vector.broadcast %0 : vector<1xf16> to vector<16x1xf16> |
| 1434 | /// ``` |
| 1435 | /// |
| 1436 | /// Example 2 (same-rank broadcast, no-op): |
| 1437 | /// ``` |
| 1438 | /// %0 = "some_op"() {layout_result_0 = |
| 1439 | /// #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>} |
| 1440 | /// : () -> vector<16x1xf16> |
| 1441 | /// %1 = vector.broadcast %0 {layout_result_0 = |
| 1442 | /// #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>} |
| 1443 | /// : vector<16x1xf16> to vector<16x16xf16> |
| 1444 | /// ``` |
| 1445 | /// is distributed to (no-op, source already matches distributed result type): |
| 1446 | /// ``` |
| 1447 | /// %0 = "some_op"() : () -> vector<16x1xf16> |
| 1448 | /// // broadcast is eliminated, %0 is used directly |
| 1449 | /// ``` |
| 1450 | /// |
| 1451 | /// Example 3 (scalar to vector broadcast): |
| 1452 | /// ``` |
| 1453 | /// %0 = "some_op"() : () -> f16 |
| 1454 | /// %1 = vector.broadcast %0 {layout_result_0 = |
| 1455 | /// #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>} |
| 1456 | /// : f16 to vector<16x16xf16> |
| 1457 | /// ``` |
| 1458 | /// is distributed to: |
| 1459 | /// ``` |
| 1460 | /// %0 = "some_op"() : f16 |
| 1461 | /// %1 = vector.broadcast %0 : f16 to vector<16x1xf16> |
| 1462 | /// ``` |
| 1463 | struct SgToLaneBroadcast : public OpConversionPattern<vector::BroadcastOp> { |
| 1464 | using OpConversionPattern<vector::BroadcastOp>::OpConversionPattern; |
| 1465 | |
| 1466 | LogicalResult |
| 1467 | matchAndRewrite(vector::BroadcastOp op, OpAdaptor adaptor, |
| 1468 | ConversionPatternRewriter &rewriter) const override { |
| 1469 | xegpu::DistributeLayoutAttr resultLayout = |
| 1470 | xegpu::getTemporaryLayout(operandOrResult: cast<OpResult>(Val: op.getResult())); |
| 1471 | if (!resultLayout || !resultLayout.isForSubgroup()) |
| 1472 | return rewriter.notifyMatchFailure( |
| 1473 | arg&: op, msg: "result does not have subgroup distribute layout" ); |
| 1474 | |
| 1475 | VectorType destType = op.getResultVectorType(); |
| 1476 | VectorType sourceType = dyn_cast<VectorType>(Val: op.getSourceType()); |
| 1477 | |
| 1478 | xegpu::DistributeLayoutAttr sourceLayout = |
| 1479 | xegpu::getTemporaryLayout(operandOrResult: op->getOpOperand(idx: 0)); |
| 1480 | |
| 1481 | if (sourceType) { |
| 1482 | int64_t rankDiff = destType.getRank() - sourceType.getRank(); |
| 1483 | if (rankDiff > 0) { |
| 1484 | // Case 1: Low-rank to high-rank broadcast. |
| 1485 | if (!sourceLayout || !sourceLayout.isSliceOf(other: resultLayout)) |
| 1486 | op.emitWarning( |
| 1487 | message: "broadcast source layout must be a slice of result layout" ); |
| 1488 | } else if (rankDiff == 0) { |
| 1489 | // Case 2: Same-rank broadcast. |
| 1490 | auto broadcastUnitDimsSet = op.computeBroadcastedUnitDims(); |
| 1491 | SmallVector<int64_t> broadcastUnitDims(broadcastUnitDimsSet.begin(), |
| 1492 | broadcastUnitDimsSet.end()); |
| 1493 | assert(sourceLayout.isEqualTo( |
| 1494 | sourceLayout.setUnitDimData(broadcastUnitDims)) && |
| 1495 | "The sg_data for unit dimensions should be set as 1" ); |
| 1496 | sourceLayout = sourceLayout.setUnitDimLayout(broadcastUnitDims); |
| 1497 | } |
| 1498 | } else { |
| 1499 | // Case 3: Scalar to vector broadcast. |
| 1500 | if (sourceLayout) |
| 1501 | return rewriter.notifyMatchFailure( |
| 1502 | arg&: op, msg: "broadcast from scalar must not have a layout attribute" ); |
| 1503 | } |
| 1504 | |
| 1505 | auto destDistType = |
| 1506 | xegpu::getDistVecTypeBasedOnLaneLayout(layout: resultLayout, originalType: destType); |
| 1507 | if (failed(Result: destDistType)) |
| 1508 | return rewriter.notifyMatchFailure( |
| 1509 | arg&: op, msg: "failed to distribute the result vector type" ); |
| 1510 | |
| 1511 | Value source = adaptor.getSource(); |
| 1512 | // If the adapted source already matches the dest dist type, it's a no-op. |
| 1513 | if (source.getType() == destDistType.value()) { |
| 1514 | rewriter.replaceOp(op, newValues: source); |
| 1515 | return success(); |
| 1516 | } |
| 1517 | |
| 1518 | auto newOp = vector::BroadcastOp::create(builder&: rewriter, location: op.getLoc(), |
| 1519 | result: destDistType.value(), source); |
| 1520 | rewriter.replaceOp(op, newOp); |
| 1521 | return success(); |
| 1522 | } |
| 1523 | }; |
| 1524 | |
| 1525 | /// Distributes a subgroup-level vector.insert_strided_slice op to |
| 1526 | /// lane-level. If the dest is distributed, the offsets are adjusted to |
| 1527 | /// match the distributed types. |
| 1528 | struct SgToLaneVectorInsertStridedSlice |
| 1529 | : public OpConversionPattern<vector::InsertStridedSliceOp> { |
| 1530 | using OpConversionPattern<vector::InsertStridedSliceOp>::OpConversionPattern; |
| 1531 | |
| 1532 | LogicalResult |
| 1533 | matchAndRewrite(vector::InsertStridedSliceOp op, OpAdaptor adaptor, |
| 1534 | ConversionPatternRewriter &rewriter) const override { |
| 1535 | xegpu::DistributeLayoutAttr resultLayout = |
| 1536 | xegpu::getTemporaryLayout(operandOrResult: op->getOpResult(idx: 0)); |
| 1537 | if (!resultLayout || !resultLayout.isForSubgroup()) |
| 1538 | return failure(); |
| 1539 | |
| 1540 | VectorType destType = op.getDestVectorType(); |
| 1541 | auto distDestTyOrFailure = |
| 1542 | xegpu::getDistVecTypeBasedOnLaneLayout(layout: resultLayout, originalType: destType); |
| 1543 | if (failed(Result: distDestTyOrFailure)) |
| 1544 | return rewriter.notifyMatchFailure( |
| 1545 | arg&: op, msg: "unable to compute distributed vector type from lane layout" ); |
| 1546 | VectorType distDestTy = *distDestTyOrFailure; |
| 1547 | |
| 1548 | SmallVector<int64_t> destDistributedDims = |
| 1549 | getDistributedDims(originalType: destType, distributedType: distDestTy); |
| 1550 | |
| 1551 | SmallVector<Attribute> updatedOffsets = llvm::map_to_vector( |
| 1552 | C: op.getOffsets(), F: [](Attribute attr) { return attr; }); |
| 1553 | |
| 1554 | if (!destDistributedDims.empty()) { |
| 1555 | if (destDistributedDims.size() != 1) |
| 1556 | return rewriter.notifyMatchFailure( |
| 1557 | arg&: op, msg: "only single dimension distribution is supported" ); |
| 1558 | int64_t destDistDim = destDistributedDims[0]; |
| 1559 | |
| 1560 | VectorType srcType = op.getSourceVectorType(); |
| 1561 | // The distributed dim must be in the last k (source rank) dims of dest. |
| 1562 | int64_t sourceDistDim = |
| 1563 | destDistDim - (destType.getRank() - srcType.getRank()); |
| 1564 | if (sourceDistDim < 0) |
| 1565 | return rewriter.notifyMatchFailure( |
| 1566 | arg&: op, msg: "distributed dimension must be in the last k dims of dest" ); |
| 1567 | |
| 1568 | auto destLayout = xegpu::getTemporaryLayout(operandOrResult: op->getOpOperand(idx: 1)); |
| 1569 | auto sourceLayout = xegpu::getTemporaryLayout(operandOrResult: op->getOpOperand(idx: 0)); |
| 1570 | if (!destLayout || !sourceLayout || |
| 1571 | destLayout.getEffectiveLaneLayoutAsInt().empty() || |
| 1572 | sourceLayout.getEffectiveLaneLayoutAsInt().empty()) |
| 1573 | return rewriter.notifyMatchFailure( |
| 1574 | arg&: op, msg: "source or dest of insert_strided_slice lacks distribution " |
| 1575 | "layout" ); |
| 1576 | |
| 1577 | auto destLaneData = destLayout.getEffectiveLaneDataAsInt(); |
| 1578 | auto sourceLaneData = sourceLayout.getEffectiveLaneDataAsInt(); |
| 1579 | // Only check lane_data for the distributed dimension. Non-distributed |
| 1580 | // dimensions may have non-unit lane_data (e.g., packed layouts). |
| 1581 | if ((destDistDim < static_cast<int64_t>(destLaneData.size()) && |
| 1582 | destLaneData[destDistDim] != 1) || |
| 1583 | (sourceDistDim < static_cast<int64_t>(sourceLaneData.size()) && |
| 1584 | sourceLaneData[sourceDistDim] != 1)) |
| 1585 | return rewriter.notifyMatchFailure( |
| 1586 | arg&: op, msg: "expecting unit lane data along the distributed dimension" ); |
| 1587 | |
| 1588 | // The distributed dimension may span only a subset of the subgroup, so |
| 1589 | // divide its size and offset by the lanes that actually cover it. |
| 1590 | auto destLaneLayout = destLayout.getEffectiveLaneLayoutAsInt(); |
| 1591 | int64_t numLanesAlongDim = |
| 1592 | destDistDim < static_cast<int64_t>(destLaneLayout.size()) |
| 1593 | ? destLaneLayout[destDistDim] |
| 1594 | : 1; |
| 1595 | |
| 1596 | int64_t srcDistrDimSize = srcType.getDimSize(idx: sourceDistDim); |
| 1597 | if (srcDistrDimSize % numLanesAlongDim != 0) |
| 1598 | return rewriter.notifyMatchFailure( |
| 1599 | arg&: op, msg: "source distributed dim size is not a multiple of " |
| 1600 | "the number of lanes along that dimension" ); |
| 1601 | |
| 1602 | int64_t destDistrDimOffset = |
| 1603 | cast<IntegerAttr>(Val: op.getOffsets()[destDistDim]).getInt(); |
| 1604 | if (destDistrDimOffset % numLanesAlongDim != 0) |
| 1605 | return rewriter.notifyMatchFailure( |
| 1606 | arg&: op, msg: "offset along distributed dim is not a multiple of " |
| 1607 | "the number of lanes along that dimension" ); |
| 1608 | // Adjust offset for the distributed dimension. |
| 1609 | updatedOffsets[destDistDim] = |
| 1610 | rewriter.getI64IntegerAttr(value: destDistrDimOffset / numLanesAlongDim); |
| 1611 | } |
| 1612 | |
| 1613 | auto newOp = vector::InsertStridedSliceOp::create( |
| 1614 | builder&: rewriter, location: op.getLoc(), result: distDestTy, valueToStore: adaptor.getValueToStore(), |
| 1615 | dest: adaptor.getDest(), |
| 1616 | offsets: ArrayAttr::get(context: rewriter.getContext(), value: updatedOffsets), strides: op.getStrides()); |
| 1617 | rewriter.replaceOp(op, newValues: newOp.getResult()); |
| 1618 | return success(); |
| 1619 | } |
| 1620 | }; |
| 1621 | |
| 1622 | /// Distributes a subgroup-level vector.insert op to lane-level. Only |
| 1623 | /// handles sub-vector insertion (value to store is VectorType, not scalar). |
| 1624 | struct SgToLaneVectorInsert : public OpConversionPattern<vector::InsertOp> { |
| 1625 | using OpConversionPattern<vector::InsertOp>::OpConversionPattern; |
| 1626 | |
| 1627 | LogicalResult |
| 1628 | matchAndRewrite(vector::InsertOp op, OpAdaptor adaptor, |
| 1629 | ConversionPatternRewriter &rewriter) const override { |
| 1630 | // Only handle vector value-to-store (not scalar insertion). |
| 1631 | auto valueType = dyn_cast<VectorType>(Val: op.getValueToStoreType()); |
| 1632 | if (!valueType) |
| 1633 | return rewriter.notifyMatchFailure(arg&: op, msg: "scalar insert not supported" ); |
| 1634 | |
| 1635 | xegpu::DistributeLayoutAttr layout = |
| 1636 | xegpu::getTemporaryLayout(operandOrResult: op->getOpResult(idx: 0)); |
| 1637 | if (!layout || !layout.isForSubgroup()) |
| 1638 | return failure(); |
| 1639 | |
| 1640 | // verify that the outer k dimensions (for offsets) |
| 1641 | // don't have non-unit lane_layout. |
| 1642 | auto laneLayout = layout.getEffectiveLaneLayoutAsInt(); |
| 1643 | if (llvm::any_of(Range: ArrayRef<int64_t>(laneLayout).drop_back(N: 1), |
| 1644 | P: [](int64_t v) { return v != 1; })) |
| 1645 | return rewriter.notifyMatchFailure( |
| 1646 | arg&: op, msg: "only innermost dimension distribution is supported for " |
| 1647 | "vector.insert" ); |
| 1648 | |
| 1649 | auto newOp = vector::InsertOp::create( |
| 1650 | builder&: rewriter, location: op.getLoc(), valueToStore: adaptor.getValueToStore(), dest: adaptor.getDest(), |
| 1651 | position: op.getMixedPosition()); |
| 1652 | rewriter.replaceOp(op, newValues: newOp.getResult()); |
| 1653 | return success(); |
| 1654 | } |
| 1655 | }; |
| 1656 | |
| 1657 | /// Redistributes `src` for a `convert_layout` that changes only the |
| 1658 | /// `lane_layout` along the outer (distributed) dimension, shrinking it from |
| 1659 | /// `currentLaneNum` to `targetLaneNum` lanes (a partial-subgroup |
| 1660 | /// distribution). Because the data is no longer replicated across all lanes, |
| 1661 | /// each surviving lane must gather the values that previously lived in the |
| 1662 | /// lanes that are dropped. The values are gathered with `gpu.shuffle` and |
| 1663 | /// concatenated with the lane-local data using `vector.shuffle`, which doubles |
| 1664 | /// the distributed outer dimension when the lane count is halved. |
| 1665 | /// |
| 1666 | /// Only halving the lane count (a factor of two) is currently supported. |
| 1667 | /// Returns the redistributed value on success, or failure if `src` cannot be |
| 1668 | /// shuffled (e.g. it is not a rank-2 vector or its bit width is not a multiple |
| 1669 | /// of 32). |
| 1670 | static FailureOr<Value> |
| 1671 | shuffleDataAsLaneLayoutChange(ConversionPatternRewriter &rewriter, Location loc, |
| 1672 | Value src, int64_t currentLaneNum, |
| 1673 | int64_t targetLaneNum) { |
| 1674 | VectorType srcTy = dyn_cast<VectorType>(Val: src.getType()); |
| 1675 | if (!srcTy || srcTy.getRank() != 2) |
| 1676 | return failure(); |
| 1677 | // Only halving the lane count (factor of two) is supported for now. |
| 1678 | if (targetLaneNum <= 0 || currentLaneNum != targetLaneNum * 2) |
| 1679 | return failure(); |
| 1680 | // gpu.shuffle operates on i32, so the data must be a multiple of 32 bits. |
| 1681 | int64_t vectorBitWidth = |
| 1682 | srcTy.getNumElements() * srcTy.getElementTypeBitWidth(); |
| 1683 | if (vectorBitWidth % 32 != 0) |
| 1684 | return failure(); |
| 1685 | |
| 1686 | // A vector cannot be shuffled across lanes directly: |
| 1687 | // -- cast the source to a 1D vector of i32 |
| 1688 | // -- create a temp 1D vector of i32 initialized to zero |
| 1689 | // -- for each i32 element: |
| 1690 | // ---- extract it from the source bundle |
| 1691 | // ---- gpu.shuffle to gather the value from the partner lane |
| 1692 | // ---- insert it into the temp bundle |
| 1693 | // -- cast the temp back to the source vector type |
| 1694 | // -- vector.shuffle the source and temp to concatenate along the outer dim |
| 1695 | Type shuffleElemTy = rewriter.getI32Type(); |
| 1696 | int64_t numShuffles = vectorBitWidth / 32; |
| 1697 | VectorType shuffleBundleTy = VectorType::get(shape: {numShuffles}, elementType: shuffleElemTy); |
| 1698 | // Initialize temp to zero. |
| 1699 | Value temp = arith::ConstantOp::create( |
| 1700 | builder&: rewriter, location: loc, |
| 1701 | value: DenseElementsAttr::get(type: shuffleBundleTy, |
| 1702 | values: IntegerAttr::get(type: shuffleElemTy, value: 0))); |
| 1703 | VectorType flatSrcTy = |
| 1704 | VectorType::get(shape: {srcTy.getNumElements()}, elementType: srcTy.getElementType()); |
| 1705 | Value flatSrc = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: flatSrcTy, source: src); |
| 1706 | Value shuffleBundle = |
| 1707 | vector::BitCastOp::create(builder&: rewriter, location: loc, result: shuffleBundleTy, source: flatSrc); |
| 1708 | for (int64_t i = 0; i < numShuffles; i++) { |
| 1709 | Value shuffleElem = |
| 1710 | vector::ExtractOp::create(builder&: rewriter, location: loc, source: shuffleBundle, position: i); |
| 1711 | shuffleElem = gpu::ShuffleOp::create(builder&: rewriter, location: loc, value: shuffleElem, offset: 0, |
| 1712 | width: targetLaneNum, mode: gpu::ShuffleMode::UP) |
| 1713 | .getResult(i: 0); |
| 1714 | temp = vector::InsertOp::create(builder&: rewriter, location: loc, valueToStore: shuffleElem, dest: temp, position: i); |
| 1715 | } |
| 1716 | temp = vector::BitCastOp::create(builder&: rewriter, location: loc, result: flatSrcTy, source: temp); |
| 1717 | temp = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: srcTy, source: temp); |
| 1718 | |
| 1719 | // Concatenate the lane-local and gathered data along the outer dimension. |
| 1720 | SmallVector<int64_t> indices(srcTy.getShape()[0] * 2); |
| 1721 | std::iota(first: indices.begin(), last: indices.end(), value: 0); |
| 1722 | Value res = vector::ShuffleOp::create(builder&: rewriter, location: loc, v1: src, v2: temp, mask: indices); |
| 1723 | return res; |
| 1724 | } |
| 1725 | |
| 1726 | /// Repacks `src`'s `lane_data` along `repackDim` between round-robin and |
| 1727 | /// contiguous form with an `xegpu.lane_shuffle`, which moves each lane's run of |
| 1728 | /// `k` elements across lanes while preserving the element type. |
| 1729 | /// |
| 1730 | /// `inputData`/`targetData` are the `repackDim` `lane_data` of the input and |
| 1731 | /// target layouts; exactly one must be 1 (round-robin) and the other `k` |
| 1732 | /// (contiguous). Returns failure if that does not hold. |
| 1733 | static FailureOr<Value> repackLaneData(ConversionPatternRewriter &rewriter, |
| 1734 | Location loc, Value src, |
| 1735 | int64_t repackDim, int64_t inputData, |
| 1736 | int64_t targetData) { |
| 1737 | auto srcTy = dyn_cast<VectorType>(Val: src.getType()); |
| 1738 | if (!srcTy) |
| 1739 | return failure(); |
| 1740 | int64_t rank = srcTy.getRank(); |
| 1741 | Type elemTy = srcTy.getElementType(); |
| 1742 | int64_t k = srcTy.getShape()[repackDim]; |
| 1743 | |
| 1744 | bool roundRobinToContig = inputData == 1 && targetData == k; |
| 1745 | bool contigToRoundRobin = inputData == k && targetData == 1; |
| 1746 | if (!roundRobinToContig && !contigToRoundRobin) |
| 1747 | return failure(); |
| 1748 | |
| 1749 | // Round-robin -> contiguous gathers a lane's strided elements into |
| 1750 | // consecutive positions (pack); the reverse scatters them back (unpack). |
| 1751 | xegpu::LaneShuffleMode mode = roundRobinToContig |
| 1752 | ? xegpu::LaneShuffleMode::Pack |
| 1753 | : xegpu::LaneShuffleMode::Unpack; |
| 1754 | VectorType runTy = VectorType::get(shape: {k}, elementType: elemTy); |
| 1755 | |
| 1756 | // Common case: the lane fragment is a single run (every dimension other than |
| 1757 | // `repackDim` is unit), so collapse it to 1D, shuffle once, and restore it. |
| 1758 | if (srcTy.getNumElements() == k) { |
| 1759 | if (rank == 1) |
| 1760 | return Value( |
| 1761 | xegpu::LaneShuffleOp::create(builder&: rewriter, location: loc, result: runTy, source: src, mode)); |
| 1762 | Value flat = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: runTy, source: src); |
| 1763 | Value shuffled = |
| 1764 | xegpu::LaneShuffleOp::create(builder&: rewriter, location: loc, result: runTy, source: flat, mode); |
| 1765 | return Value(vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: srcTy, source: shuffled)); |
| 1766 | } |
| 1767 | |
| 1768 | // When `repackDim` is innermost each run is a contiguous sub-vector, so it is |
| 1769 | // extracted and re-inserted as a whole. |
| 1770 | if (repackDim == rank - 1) { |
| 1771 | SmallVector<int64_t> outerShape(srcTy.getShape().drop_back()); |
| 1772 | int64_t numRuns = computeProduct(basis: outerShape); |
| 1773 | SmallVector<int64_t> outerStrides = computeStrides(sizes: outerShape); |
| 1774 | Value result = arith::ConstantOp::create(builder&: rewriter, location: loc, result: srcTy, |
| 1775 | value: rewriter.getZeroAttr(type: srcTy)); |
| 1776 | for (int64_t i = 0; i < numRuns; ++i) { |
| 1777 | SmallVector<int64_t> pos = delinearize(linearIndex: i, strides: outerStrides); |
| 1778 | Value run = vector::ExtractOp::create(builder&: rewriter, location: loc, source: src, position: pos); |
| 1779 | Value shuffled = |
| 1780 | xegpu::LaneShuffleOp::create(builder&: rewriter, location: loc, result: runTy, source: run, mode); |
| 1781 | result = vector::InsertOp::create(builder&: rewriter, location: loc, valueToStore: shuffled, dest: result, position: pos); |
| 1782 | } |
| 1783 | return result; |
| 1784 | } |
| 1785 | |
| 1786 | // Otherwise each run is strided along `repackDim`: extract the `k`-long slice |
| 1787 | // (a sub-vector that is unit along every other dim), flatten it to 1D, |
| 1788 | // shuffle, and insert it back. |
| 1789 | SmallVector<int64_t> keptShape; |
| 1790 | SmallVector<int64_t> keptDims; |
| 1791 | for (int64_t d = 0; d < rank; ++d) |
| 1792 | if (d != repackDim) { |
| 1793 | keptShape.push_back(Elt: srcTy.getShape()[d]); |
| 1794 | keptDims.push_back(Elt: d); |
| 1795 | } |
| 1796 | int64_t numRuns = computeProduct(basis: keptShape); |
| 1797 | SmallVector<int64_t> keptStrides = computeStrides(sizes: keptShape); |
| 1798 | SmallVector<int64_t> sliceSizes(rank, 1); |
| 1799 | sliceSizes[repackDim] = k; |
| 1800 | SmallVector<int64_t> sliceStrides(rank, 1); |
| 1801 | VectorType sliceTy = VectorType::get(shape: sliceSizes, elementType: elemTy); |
| 1802 | Value result = arith::ConstantOp::create(builder&: rewriter, location: loc, result: srcTy, |
| 1803 | value: rewriter.getZeroAttr(type: srcTy)); |
| 1804 | for (int64_t i = 0; i < numRuns; ++i) { |
| 1805 | SmallVector<int64_t> keptPos = delinearize(linearIndex: i, strides: keptStrides); |
| 1806 | SmallVector<int64_t> offsets(rank, 0); |
| 1807 | for (auto [dim, coord] : llvm::zip_equal(t&: keptDims, u&: keptPos)) |
| 1808 | offsets[dim] = coord; |
| 1809 | Value slice = vector::ExtractStridedSliceOp::create( |
| 1810 | builder&: rewriter, location: loc, source: src, offsets, sizes: sliceSizes, strides: sliceStrides); |
| 1811 | Value run = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: runTy, source: slice); |
| 1812 | Value repacked = |
| 1813 | xegpu::LaneShuffleOp::create(builder&: rewriter, location: loc, result: runTy, source: run, mode); |
| 1814 | Value repackedSlice = |
| 1815 | vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: sliceTy, source: repacked); |
| 1816 | result = vector::InsertStridedSliceOp::create( |
| 1817 | builder&: rewriter, location: loc, valueToStore: repackedSlice, dest: result, offsets, strides: sliceStrides); |
| 1818 | } |
| 1819 | return result; |
| 1820 | } |
| 1821 | |
| 1822 | /// Folds a subgroup-level ConvertLayout op with compatible lane layouts. |
| 1823 | struct SgToLaneConvertLayout |
| 1824 | : public OpConversionPattern<xegpu::ConvertLayoutOp> { |
| 1825 | using OpConversionPattern<xegpu::ConvertLayoutOp>::OpConversionPattern; |
| 1826 | |
| 1827 | LogicalResult |
| 1828 | matchAndRewrite(xegpu::ConvertLayoutOp op, OpAdaptor adaptor, |
| 1829 | ConversionPatternRewriter &rewriter) const override { |
| 1830 | auto inputLayout = op.getEffectiveInputLayout(); |
| 1831 | auto targetLayout = op.getTargetLayoutAttr(); |
| 1832 | Type valType = op.getResult().getType(); |
| 1833 | |
| 1834 | if (valType.isIntOrFloat()) { |
| 1835 | rewriter.replaceOp(op, newValues: op.getSource()); |
| 1836 | return success(); |
| 1837 | } |
| 1838 | |
| 1839 | auto resShape = cast<VectorType>(Val&: valType).getShape(); |
| 1840 | SmallVector<int64_t> resShapeVec(resShape.begin(), resShape.end()); |
| 1841 | |
| 1842 | // Equivalent layouts: the convert_layout is a no-op and folds to its |
| 1843 | // source. |
| 1844 | if (inputLayout.isCompatibleWith(other: targetLayout, shape: resShapeVec, |
| 1845 | level: xegpu::LayoutKind::Lane)) { |
| 1846 | rewriter.replaceOp(op, newValues: adaptor.getSource()); |
| 1847 | return success(); |
| 1848 | } |
| 1849 | |
| 1850 | // Handle the special case where the conversion redistributes a value |
| 1851 | // across a fraction of the subgroup: the lane_layout shrinks along the |
| 1852 | // outer (distributed) dimension while lane_data stays the same. Only a |
| 1853 | // pure outer-dimension lane_layout change is supported, so the inner |
| 1854 | // lane_layout must be unit (making the outer dim the only distributed one) |
| 1855 | // and the outer lane_layout must be genuinely distributed (> 1), which |
| 1856 | // also rules out the degenerate [1, 1] layout. |
| 1857 | if (inputLayout.getEffectiveOrderAsInt() == |
| 1858 | targetLayout.getEffectiveOrderAsInt() && |
| 1859 | inputLayout.getRank() == 2 && targetLayout.getRank() == 2) { |
| 1860 | auto laneLayout = inputLayout.getEffectiveLaneLayoutAsInt(); |
| 1861 | auto targetLaneLayout = targetLayout.getEffectiveLaneLayoutAsInt(); |
| 1862 | auto laneData = inputLayout.getEffectiveLaneDataAsInt(); |
| 1863 | auto targetLaneData = targetLayout.getEffectiveLaneDataAsInt(); |
| 1864 | if (laneLayout.size() == 2 && targetLaneLayout.size() == 2 && |
| 1865 | laneData == targetLaneData && laneLayout[1] == 1 && |
| 1866 | targetLaneLayout[1] == 1 && laneLayout[0] > 1 && |
| 1867 | laneLayout[0] != targetLaneLayout[0]) { |
| 1868 | FailureOr<Value> res = shuffleDataAsLaneLayoutChange( |
| 1869 | rewriter, loc: op.getLoc(), src: adaptor.getSource(), currentLaneNum: laneLayout[0], |
| 1870 | targetLaneNum: targetLaneLayout[0]); |
| 1871 | if (succeeded(Result: res)) { |
| 1872 | rewriter.replaceOp(op, newValues: *res); |
| 1873 | return success(); |
| 1874 | } |
| 1875 | } |
| 1876 | } |
| 1877 | |
| 1878 | // Handle a pure `lane_data` repack: `lane_layout` and `order` are unchanged |
| 1879 | // and exactly one dimension's `lane_data` switches between round-robin |
| 1880 | // (lane_data 1) and contiguous (lane_data == run length). The elements per |
| 1881 | // lane are unchanged, but their assignment to lanes is not, so the data is |
| 1882 | // moved across lanes with `xegpu.lane_shuffle`. The changed dimension must |
| 1883 | // be one of the two innermost ones, since sg-to-lane distribution is 2D. |
| 1884 | if (inputLayout.getEffectiveOrderAsInt() == |
| 1885 | targetLayout.getEffectiveOrderAsInt() && |
| 1886 | inputLayout.getEffectiveLaneLayoutAsInt() == |
| 1887 | targetLayout.getEffectiveLaneLayoutAsInt()) { |
| 1888 | auto laneLayout = inputLayout.getEffectiveLaneLayoutAsInt(); |
| 1889 | auto laneData = inputLayout.getEffectiveLaneDataAsInt(); |
| 1890 | auto targetLaneData = targetLayout.getEffectiveLaneDataAsInt(); |
| 1891 | // Find the single dimension whose lane_data changed; bail out if more |
| 1892 | // than one differs. |
| 1893 | int64_t rank = laneData.size(); |
| 1894 | int64_t repackDim = -1; |
| 1895 | bool multipleChanged = false; |
| 1896 | for (int64_t d = 0; d < rank; ++d) |
| 1897 | if (laneData[d] != targetLaneData[d]) { |
| 1898 | if (repackDim != -1) |
| 1899 | multipleChanged = true; |
| 1900 | repackDim = d; |
| 1901 | } |
| 1902 | |
| 1903 | // `repackDim` must be the distributed dim (lane_layout != 1) and the |
| 1904 | // other innermost dim non-distributed (lane_layout == 1). |
| 1905 | int64_t otherDim = repackDim == rank - 1 ? rank - 2 : rank - 1; |
| 1906 | bool laneLayoutOk = repackDim != -1 && laneLayout[repackDim] != 1 && |
| 1907 | (rank < 2 || laneLayout[otherDim] == 1); |
| 1908 | |
| 1909 | // Exactly one dimension must change, and it must be one of the two |
| 1910 | // innermost (>= rank - 2). |
| 1911 | if (repackDim != -1 && repackDim >= rank - 2 && !multipleChanged && |
| 1912 | laneLayoutOk) { |
| 1913 | FailureOr<Value> res = repackLaneData( |
| 1914 | rewriter, loc: op.getLoc(), src: adaptor.getSource(), repackDim, |
| 1915 | inputData: laneData[repackDim], targetData: targetLaneData[repackDim]); |
| 1916 | if (succeeded(Result: res)) { |
| 1917 | rewriter.replaceOp(op, newValues: *res); |
| 1918 | return success(); |
| 1919 | } |
| 1920 | } |
| 1921 | } |
| 1922 | |
| 1923 | return rewriter.notifyMatchFailure( |
| 1924 | arg&: op, msg: "lowering incompatible convert_layout not yet supported" ); |
| 1925 | } |
| 1926 | }; |
| 1927 | |
| 1928 | /// The offsets and shape of one `lane_data`-sized piece of a lane's fragment. |
| 1929 | /// `computeDistributedCoords` returns one coordinate per distribution unit, |
| 1930 | /// enumerated row major over `shape / (lane_layout * lane_data)`, and unit `u` |
| 1931 | /// owns the piece of the fragment at `delinearize(u) * lane_data`. |
| 1932 | struct LaneFragmentPiece { |
| 1933 | SmallVector<int64_t> offsets; |
| 1934 | SmallVector<int64_t> sizes; |
| 1935 | }; |
| 1936 | |
| 1937 | static LaneFragmentPiece getLaneFragmentPiece(int64_t unit, |
| 1938 | ArrayRef<int64_t> shape, |
| 1939 | ArrayRef<int64_t> laneLayout, |
| 1940 | ArrayRef<int64_t> laneData) { |
| 1941 | int64_t rank = shape.size(); |
| 1942 | SmallVector<int64_t> units(rank); |
| 1943 | for (int64_t d = 0; d < rank; ++d) |
| 1944 | units[d] = shape[d] / std::min(a: shape[d], b: laneLayout[d] * laneData[d]); |
| 1945 | SmallVector<int64_t> unitCoords = delinearize(linearIndex: unit, strides: computeStrides(sizes: units)); |
| 1946 | LaneFragmentPiece piece; |
| 1947 | for (int64_t d = 0; d < rank; ++d) { |
| 1948 | piece.offsets.push_back(Elt: unitCoords[d] * laneData[d]); |
| 1949 | piece.sizes.push_back(Elt: laneData[d]); |
| 1950 | } |
| 1951 | return piece; |
| 1952 | } |
| 1953 | |
| 1954 | /// Last-resort lowering for a `convert_layout` that none of the patterns above |
| 1955 | /// handle: round-trip the value through shared local memory, writing each |
| 1956 | /// lane's fragment to the coordinates `input_layout` gives it and reading it |
| 1957 | /// back from the ones `target_layout` gives it. Any pair of layouts that can |
| 1958 | /// both distribute the value works, at the cost of an SLM write and read. |
| 1959 | /// |
| 1960 | /// The op is subgroup level, but every subgroup of the workgroup executes it on |
| 1961 | /// its own data, so the scratch holds one tile per subgroup and each subgroup |
| 1962 | /// addresses its own through a `gpu.subgroup_id` offset on the outermost |
| 1963 | /// dimension. The subgroup count comes from the kernel's `known_block_size`. |
| 1964 | /// |
| 1965 | /// A lane's fragment is one `lane_data`-sized piece per distribution unit, so |
| 1966 | /// the transfer is one `store_matrix`/`load_matrix` per unit; layouts with |
| 1967 | /// a single unit, which is the common case, give a single pair. |
| 1968 | struct SgToLaneConvertLayoutViaSLM |
| 1969 | : public OpConversionPattern<xegpu::ConvertLayoutOp> { |
| 1970 | using OpConversionPattern<xegpu::ConvertLayoutOp>::OpConversionPattern; |
| 1971 | |
| 1972 | LogicalResult |
| 1973 | matchAndRewrite(xegpu::ConvertLayoutOp op, OpAdaptor adaptor, |
| 1974 | ConversionPatternRewriter &rewriter) const override { |
| 1975 | xegpu::DistributeLayoutAttr inputLayout = op.getEffectiveInputLayout(); |
| 1976 | xegpu::DistributeLayoutAttr targetLayout = op.getTargetLayoutAttr(); |
| 1977 | if (!inputLayout || !targetLayout || !inputLayout.isForSubgroup() || |
| 1978 | !targetLayout.isForSubgroup()) |
| 1979 | return rewriter.notifyMatchFailure(arg&: op, msg: "both layouts must be lane level" ); |
| 1980 | |
| 1981 | auto valueTy = dyn_cast<VectorType>(Val: op.getResult().getType()); |
| 1982 | if (!valueTy) |
| 1983 | return rewriter.notifyMatchFailure(arg&: op, msg: "value type must be a vector" ); |
| 1984 | Type elemTy = valueTy.getElementType(); |
| 1985 | // The scratch is sized in bytes, so sub-byte elements would not get a |
| 1986 | // well-defined address. |
| 1987 | if (!elemTy.isIntOrFloat() || elemTy.getIntOrFloatBitWidth() % 8 != 0) |
| 1988 | return rewriter.notifyMatchFailure( |
| 1989 | arg&: op, msg: "element type must be a whole number of bytes" ); |
| 1990 | |
| 1991 | ArrayRef<int64_t> sgShape = valueTy.getShape(); |
| 1992 | int64_t rank = sgShape.size(); |
| 1993 | if (rank != inputLayout.getRank() || rank != targetLayout.getRank()) |
| 1994 | return rewriter.notifyMatchFailure( |
| 1995 | arg&: op, msg: "both layouts must have the rank of the value" ); |
| 1996 | |
| 1997 | FailureOr<VectorType> distInputTy = |
| 1998 | xegpu::getDistVecTypeBasedOnLaneLayout(layout: inputLayout, originalType: valueTy); |
| 1999 | FailureOr<VectorType> distTargetTy = |
| 2000 | xegpu::getDistVecTypeBasedOnLaneLayout(layout: targetLayout, originalType: valueTy); |
| 2001 | if (failed(Result: distInputTy) || failed(Result: distTargetTy)) |
| 2002 | return rewriter.notifyMatchFailure( |
| 2003 | arg&: op, msg: "value type must be distributable by both layouts" ); |
| 2004 | |
| 2005 | const auto *uArch = |
| 2006 | xegpu::uArch::getUArch(archName: xegpu::getChipStr(op).value_or(u: "" )); |
| 2007 | if (!uArch) |
| 2008 | return rewriter.notifyMatchFailure( |
| 2009 | arg&: op, msg: "target attribute is required to determine the subgroup size" ); |
| 2010 | FailureOr<int64_t> numSubgroups = |
| 2011 | xegpu::getNumSubgroupsFromBlockSize(op, subgroupSize: uArch->getSubgroupSize()); |
| 2012 | if (failed(Result: numSubgroups)) |
| 2013 | return rewriter.notifyMatchFailure( |
| 2014 | arg&: op, msg: "the scratch holds one tile per subgroup, so @known_block_size " |
| 2015 | "must be attached to the kernel, with power-of-two dimensions " |
| 2016 | "covering at least one subgroup" ); |
| 2017 | |
| 2018 | Location loc = op.getLoc(); |
| 2019 | |
| 2020 | // One tile per subgroup, stacked along the outermost dimension. |
| 2021 | SmallVector<int64_t> slmShape(sgShape); |
| 2022 | slmShape[0] *= *numSubgroups; |
| 2023 | int64_t slmBytes = |
| 2024 | computeProduct(basis: slmShape) * elemTy.getIntOrFloatBitWidth() / 8; |
| 2025 | auto slmTy = MemRefType::get(shape: {slmBytes}, elementType: rewriter.getI8Type(), map: {}, memorySpaceInd: 3); |
| 2026 | Value slm = memref::AllocaOp::create(builder&: rewriter, location: loc, memrefType: slmTy); |
| 2027 | Value memDesc = xegpu::CreateMemDescOp::create( |
| 2028 | builder&: rewriter, location: loc, |
| 2029 | mem_desc: xegpu::MemDescType::get(context: rewriter.getContext(), shape: slmShape, elementType: elemTy, |
| 2030 | mem_layout: nullptr), |
| 2031 | source: slm); |
| 2032 | |
| 2033 | // This subgroup's tile starts at `subgroup_id` tiles into the scratch. |
| 2034 | Value sgId = gpu::SubgroupIdOp::create(builder&: rewriter, location: loc, |
| 2035 | result: rewriter.getIndexType(), upper_bound: nullptr); |
| 2036 | SmallVector<Value> base(rank); |
| 2037 | base[0] = arith::MulIOp::create( |
| 2038 | builder&: rewriter, location: loc, lhs: sgId, |
| 2039 | rhs: arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: sgShape[0])); |
| 2040 | for (int64_t d = 1; d < rank; ++d) |
| 2041 | base[d] = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0); |
| 2042 | |
| 2043 | Value laneId = gpu::LaneIdOp::create(builder&: rewriter, location: loc, result: rewriter.getIndexType(), |
| 2044 | /*upperBound=*/upper_bound: mlir::IntegerAttr()); |
| 2045 | auto dynamicOffsets = rewriter.getDenseI64ArrayAttr( |
| 2046 | values: SmallVector<int64_t>(rank, ShapedType::kDynamic)); |
| 2047 | |
| 2048 | // Adds this subgroup's base to one distribution unit's coordinates. |
| 2049 | auto withBase = [&](ArrayRef<Value> coords) { |
| 2050 | SmallVector<Value> offsets; |
| 2051 | for (auto [coord, b] : llvm::zip_equal(t&: coords, u&: base)) |
| 2052 | offsets.push_back(Elt: arith::AddIOp::create(builder&: rewriter, location: loc, lhs: coord, rhs: b)); |
| 2053 | return offsets; |
| 2054 | }; |
| 2055 | |
| 2056 | // Write phase: every lane stores the pieces `input_layout` gives it. |
| 2057 | auto storeCoords = |
| 2058 | inputLayout.computeDistributedCoords(builder&: rewriter, loc, linearId: laneId, shape: sgShape); |
| 2059 | if (failed(Result: storeCoords)) |
| 2060 | return rewriter.notifyMatchFailure( |
| 2061 | arg&: op, msg: "failed to compute the input_layout coordinates" ); |
| 2062 | SmallVector<int64_t> inputLaneLayout = |
| 2063 | inputLayout.getEffectiveLaneLayoutAsInt(); |
| 2064 | SmallVector<int64_t> inputLaneData = |
| 2065 | inputLayout.getEffectiveLaneDataAsInt(); |
| 2066 | Value fragment = |
| 2067 | castValueTo(rewriter, v: cast<TypedValue<VectorType>>(Val: adaptor.getSource()), |
| 2068 | expectedTy: *distInputTy); |
| 2069 | for (auto [unit, coords] : llvm::enumerate(First&: *storeCoords)) { |
| 2070 | LaneFragmentPiece piece = |
| 2071 | getLaneFragmentPiece(unit, shape: sgShape, laneLayout: inputLaneLayout, laneData: inputLaneData); |
| 2072 | Value data = fragment; |
| 2073 | if (piece.sizes != SmallVector<int64_t>(distInputTy->getShape())) |
| 2074 | data = vector::ExtractStridedSliceOp::create( |
| 2075 | builder&: rewriter, location: loc, source: fragment, offsets: piece.offsets, sizes: piece.sizes, |
| 2076 | strides: SmallVector<int64_t>(rank, 1)); |
| 2077 | xegpu::StoreMatrixOp::create(builder&: rewriter, location: loc, resultTypes: TypeRange{}, data, mem_desc: memDesc, |
| 2078 | offsets: ValueRange(withBase(coords)), const_offsets: dynamicOffsets, |
| 2079 | subgroup_block_io: nullptr, layout: xegpu::DistributeLayoutAttr{}); |
| 2080 | } |
| 2081 | |
| 2082 | // The scratch is only exchanged between the lanes of one subgroup, which is |
| 2083 | // a single hardware thread, so ordering the write before the read is enough |
| 2084 | // and no workgroup barrier is needed. |
| 2085 | xegpu::FenceOp::create(builder&: rewriter, location: loc, memory_kind: xegpu::MemorySpace::SLM, |
| 2086 | fence_scope: xegpu::FenceScope::Workgroup); |
| 2087 | |
| 2088 | // Read phase: every lane loads the pieces `target_layout` gives it. |
| 2089 | auto loadCoords = |
| 2090 | targetLayout.computeDistributedCoords(builder&: rewriter, loc, linearId: laneId, shape: sgShape); |
| 2091 | if (failed(Result: loadCoords)) |
| 2092 | return rewriter.notifyMatchFailure( |
| 2093 | arg&: op, msg: "failed to compute the target_layout coordinates" ); |
| 2094 | SmallVector<int64_t> targetLaneLayout = |
| 2095 | targetLayout.getEffectiveLaneLayoutAsInt(); |
| 2096 | SmallVector<int64_t> targetLaneData = |
| 2097 | targetLayout.getEffectiveLaneDataAsInt(); |
| 2098 | Value result; |
| 2099 | if (loadCoords->size() > 1) |
| 2100 | result = arith::ConstantOp::create(builder&: rewriter, location: loc, result: *distTargetTy, |
| 2101 | value: rewriter.getZeroAttr(type: *distTargetTy)); |
| 2102 | for (auto [unit, coords] : llvm::enumerate(First&: *loadCoords)) { |
| 2103 | LaneFragmentPiece piece = |
| 2104 | getLaneFragmentPiece(unit, shape: sgShape, laneLayout: targetLaneLayout, laneData: targetLaneData); |
| 2105 | auto pieceTy = VectorType::get(shape: piece.sizes, elementType: elemTy); |
| 2106 | Value loaded = xegpu::LoadMatrixOp::create( |
| 2107 | builder&: rewriter, location: loc, res: pieceTy, mem_desc: memDesc, offsets: ValueRange(withBase(coords)), |
| 2108 | const_offsets: dynamicOffsets, subgroup_block_io: nullptr, layout: xegpu::DistributeLayoutAttr{}); |
| 2109 | if (!result) { |
| 2110 | result = loaded; |
| 2111 | break; |
| 2112 | } |
| 2113 | result = vector::InsertStridedSliceOp::create( |
| 2114 | builder&: rewriter, location: loc, valueToStore: loaded, dest: result, offsets: piece.offsets, |
| 2115 | strides: SmallVector<int64_t>(rank, 1)); |
| 2116 | } |
| 2117 | |
| 2118 | rewriter.replaceOp(op, newValues: castValueTo(rewriter, |
| 2119 | v: cast<TypedValue<VectorType>>(Val&: result), |
| 2120 | expectedTy: *distTargetTy)); |
| 2121 | return success(); |
| 2122 | } |
| 2123 | }; |
| 2124 | |
| 2125 | /// `getEffectiveLaneDataAsInt` is empty when `lane_data` is unset, so the unit |
| 2126 | /// check also rejects layouts that are not lane level. |
| 2127 | static bool hasDefaultOrderAndUnitLaneData(xegpu::DistributeLayoutAttr layout) { |
| 2128 | if (layout.getRank() != 2) |
| 2129 | return false; |
| 2130 | return layout.getEffectiveLaneDataAsInt() == SmallVector<int64_t>{1, 1} && |
| 2131 | layout.getEffectiveOrderAsInt() == SmallVector<int64_t>{1, 0}; |
| 2132 | } |
| 2133 | |
| 2134 | /// The quantities every element-to-lane redistribution needs, see |
| 2135 | /// `matchElementLaneRedistribution`. |
| 2136 | struct ElementLaneRedistribution { |
| 2137 | /// Subgroup-level type of the converted value. |
| 2138 | VectorType valueType; |
| 2139 | /// `valueType` as distributed by `input_layout` and by `target_layout`. |
| 2140 | VectorType distributedInput; |
| 2141 | VectorType distributedTarget; |
| 2142 | /// Effective `lane_layout` of `input_layout` and of `target_layout`. These |
| 2143 | /// always differ, otherwise the conversion would already have folded. |
| 2144 | SmallVector<int64_t> inputLaneLayout; |
| 2145 | SmallVector<int64_t> targetLaneLayout; |
| 2146 | int64_t subgroupSize; |
| 2147 | }; |
| 2148 | |
| 2149 | /// Recognizes an `xegpu.convert_layout` that redistributes individual elements |
| 2150 | /// between the lanes of a subgroup, which is the common condition the three |
| 2151 | /// slice-attributed lowerings below have. |
| 2152 | /// Both input and target layouts must satisfy `hasDefaultOrderAndUnitLaneData` |
| 2153 | /// and must be able to distribute the value, the two can only differ in which |
| 2154 | /// lane owns which element. |
| 2155 | static FailureOr<ElementLaneRedistribution> |
| 2156 | matchElementLaneRedistribution(xegpu::ConvertLayoutOp op, |
| 2157 | ConversionPatternRewriter &rewriter) { |
| 2158 | xegpu::DistributeLayoutAttr inputLayout = op.getEffectiveInputLayout(); |
| 2159 | xegpu::DistributeLayoutAttr targetLayout = op.getTargetLayoutAttr(); |
| 2160 | if (!hasDefaultOrderAndUnitLaneData(layout: inputLayout) || |
| 2161 | !hasDefaultOrderAndUnitLaneData(layout: targetLayout)) |
| 2162 | return rewriter.notifyMatchFailure( |
| 2163 | arg&: op, msg: "both layouts must be rank 2 with effective lane_data [1, 1] and " |
| 2164 | "effective order [1, 0]" ); |
| 2165 | |
| 2166 | auto valueType = dyn_cast<VectorType>(Val: op.getResult().getType()); |
| 2167 | if (!valueType) |
| 2168 | return rewriter.notifyMatchFailure(arg&: op, msg: "value type must be a vector" ); |
| 2169 | |
| 2170 | FailureOr<VectorType> distributedInput = |
| 2171 | xegpu::getDistVecTypeBasedOnLaneLayout(layout: inputLayout, originalType: valueType); |
| 2172 | FailureOr<VectorType> distributedTarget = |
| 2173 | xegpu::getDistVecTypeBasedOnLaneLayout(layout: targetLayout, originalType: valueType); |
| 2174 | if (failed(Result: distributedInput) || failed(Result: distributedTarget)) |
| 2175 | return rewriter.notifyMatchFailure( |
| 2176 | arg&: op, msg: "value type must be distributable by both layouts" ); |
| 2177 | |
| 2178 | const auto *uArch = |
| 2179 | xegpu::uArch::getUArch(archName: xegpu::getChipStr(op).value_or(u: "" )); |
| 2180 | if (!uArch) |
| 2181 | return rewriter.notifyMatchFailure( |
| 2182 | arg&: op, msg: "target attribute is required to determine the subgroup size" ); |
| 2183 | |
| 2184 | ElementLaneRedistribution redistribution; |
| 2185 | redistribution.valueType = valueType; |
| 2186 | redistribution.distributedInput = *distributedInput; |
| 2187 | redistribution.distributedTarget = *distributedTarget; |
| 2188 | redistribution.inputLaneLayout = inputLayout.getEffectiveLaneLayoutAsInt(); |
| 2189 | redistribution.targetLaneLayout = targetLayout.getEffectiveLaneLayoutAsInt(); |
| 2190 | redistribution.subgroupSize = uArch->getSubgroupSize(); |
| 2191 | return redistribution; |
| 2192 | } |
| 2193 | |
| 2194 | /// Returns the lane stride of the single distributed dimension of `slice`, i.e. |
| 2195 | /// the distance in lane ids between two adjacent coordinates along that |
| 2196 | /// dimension. `slice` must have exactly one non-sliced dimension whose parent |
| 2197 | /// `lane_layout` extent is greater than one; the stride is the product of the |
| 2198 | /// parent `lane_layout` extents that precede it in the parent `order`. |
| 2199 | /// |
| 2200 | /// Example: for |
| 2201 | /// #xegpu.slice<#xegpu.layout<lane_layout = [8, 1, 2], lane_data = [4, 1, 1], |
| 2202 | /// order = [0, 2, 1]>, dims = [0]> |
| 2203 | /// the only such dimension is parent dim 2, of extent 2. `order` makes dim 0 |
| 2204 | /// the fastest varying, with extent 8, so lanes 0..7 hold coordinate 0 and |
| 2205 | /// lanes 8..15 hold coordinate 1: the stride is 8. |
| 2206 | static FailureOr<int64_t> getDistributedDimLaneStride(xegpu::SliceAttr slice) { |
| 2207 | xegpu::SliceAttr flattened = slice.flatten(); |
| 2208 | auto parent = dyn_cast<xegpu::LayoutAttr>(Val: flattened.getParent()); |
| 2209 | if (!parent) |
| 2210 | return failure(); |
| 2211 | |
| 2212 | SmallVector<int64_t> parentLaneLayout = parent.getEffectiveLaneLayoutAsInt(); |
| 2213 | SmallVector<int64_t> parentOrder = parent.getEffectiveOrderAsInt(); |
| 2214 | if (parentLaneLayout.size() != parentOrder.size()) |
| 2215 | return failure(); |
| 2216 | |
| 2217 | ArrayRef<int64_t> slicedDims = flattened.getDims().asArrayRef(); |
| 2218 | std::optional<int64_t> distributedDim; |
| 2219 | for (int64_t dim = 0, rank = static_cast<int64_t>(parentLaneLayout.size()); |
| 2220 | dim < rank; ++dim) { |
| 2221 | if (llvm::is_contained(Range&: slicedDims, Element: dim) || parentLaneLayout[dim] == 1) |
| 2222 | continue; |
| 2223 | if (distributedDim) |
| 2224 | return failure(); |
| 2225 | distributedDim = dim; |
| 2226 | } |
| 2227 | if (!distributedDim) |
| 2228 | return failure(); |
| 2229 | |
| 2230 | int64_t stride = 1; |
| 2231 | for (int64_t dim : parentOrder) { |
| 2232 | if (dim == *distributedDim) |
| 2233 | return stride; |
| 2234 | stride *= parentLaneLayout[dim]; |
| 2235 | } |
| 2236 | return failure(); |
| 2237 | } |
| 2238 | |
| 2239 | /// Distributes the slice-attributed `xegpu.convert_layout` whose source is |
| 2240 | /// fully broadcast, i.e. every lane holds the whole value, onto the rows of a |
| 2241 | /// subset of the lanes. Each lane keeps a single element, so no data crosses |
| 2242 | /// lanes and one `vector.extract` suffices. |
| 2243 | /// |
| 2244 | /// The input layout has effective `lane_layout` [1, 1], so the distributed |
| 2245 | /// source is the whole value; the target has [n, 1], so lane `l` keeps row |
| 2246 | /// `l % n` and the distributed result is `vector<1x1>`. |
| 2247 | /// |
| 2248 | /// The source is flattened first because `xegpu-vector-linearize` cannot |
| 2249 | /// linearize a `vector.extract` with a dynamic position out of a rank-2 value. |
| 2250 | struct |
| 2251 | : public OpConversionPattern<xegpu::ConvertLayoutOp> { |
| 2252 | using OpConversionPattern<xegpu::ConvertLayoutOp>::OpConversionPattern; |
| 2253 | |
| 2254 | LogicalResult |
| 2255 | matchAndRewrite(xegpu::ConvertLayoutOp op, OpAdaptor adaptor, |
| 2256 | ConversionPatternRewriter &rewriter) const override { |
| 2257 | xegpu::DistributeLayoutAttr inputLayout = op.getEffectiveInputLayout(); |
| 2258 | xegpu::DistributeLayoutAttr targetLayout = op.getTargetLayoutAttr(); |
| 2259 | |
| 2260 | if (!isa<xegpu::SliceAttr>(Val: inputLayout)) |
| 2261 | return rewriter.notifyMatchFailure(arg&: op, |
| 2262 | msg: "input_layout must be #xegpu.slice" ); |
| 2263 | if (!isa<xegpu::LayoutAttr>(Val: targetLayout)) |
| 2264 | return rewriter.notifyMatchFailure(arg&: op, |
| 2265 | msg: "target_layout must be #xegpu.layout" ); |
| 2266 | |
| 2267 | FailureOr<ElementLaneRedistribution> redistribution = |
| 2268 | matchElementLaneRedistribution(op, rewriter); |
| 2269 | if (failed(Result: redistribution)) |
| 2270 | return failure(); |
| 2271 | |
| 2272 | if (redistribution->inputLaneLayout != SmallVector<int64_t>{1, 1}) |
| 2273 | return rewriter.notifyMatchFailure( |
| 2274 | arg&: op, msg: "input_layout effective lane_layout must be [1, 1]" ); |
| 2275 | SmallVector<int64_t> targetLaneLayout = redistribution->targetLaneLayout; |
| 2276 | if (targetLaneLayout[0] <= 1 || targetLaneLayout[1] != 1) |
| 2277 | return rewriter.notifyMatchFailure( |
| 2278 | arg&: op, msg: "target_layout effective lane_layout must be [n, 1] with n > 1" ); |
| 2279 | if (redistribution->distributedInput != redistribution->valueType) |
| 2280 | return rewriter.notifyMatchFailure( |
| 2281 | arg&: op, msg: "distributed input_layout type must equal the value type" ); |
| 2282 | if (redistribution->distributedTarget.getShape() != ArrayRef<int64_t>{1, 1}) |
| 2283 | return rewriter.notifyMatchFailure( |
| 2284 | arg&: op, msg: "distributed target_layout type must be vector<1x1>" ); |
| 2285 | if (targetLaneLayout[0] > redistribution->subgroupSize) |
| 2286 | return rewriter.notifyMatchFailure( |
| 2287 | arg&: op, msg: "target_layout effective lane_layout[0] must not exceed the " |
| 2288 | "subgroup size" ); |
| 2289 | |
| 2290 | VectorType valueType = redistribution->valueType; |
| 2291 | Location loc = op.getLoc(); |
| 2292 | Value src = |
| 2293 | castValueTo(rewriter, v: cast<TypedValue<VectorType>>(Val: adaptor.getSource()), |
| 2294 | expectedTy: redistribution->distributedInput); |
| 2295 | auto flatType = VectorType::get(shape: {valueType.getNumElements()}, |
| 2296 | elementType: valueType.getElementType()); |
| 2297 | Value flat = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: flatType, source: src); |
| 2298 | Value laneId = gpu::LaneIdOp::create(builder&: rewriter, location: loc, result: rewriter.getIndexType(), |
| 2299 | /*upperBound=*/upper_bound: mlir::IntegerAttr()); |
| 2300 | Value rowCount = |
| 2301 | arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: targetLaneLayout[0]); |
| 2302 | Value row = arith::RemUIOp::create(builder&: rewriter, location: loc, lhs: laneId, rhs: rowCount); |
| 2303 | Value element = vector::ExtractOp::create(builder&: rewriter, location: loc, source: flat, |
| 2304 | position: ArrayRef<OpFoldResult>{row}); |
| 2305 | rewriter.replaceOpWithNewOp<vector::FromElementsOp>( |
| 2306 | op, args&: redistribution->distributedTarget, args&: element); |
| 2307 | return success(); |
| 2308 | } |
| 2309 | }; |
| 2310 | |
| 2311 | /// Distributes the slice-attributed `xegpu.convert_layout` whose source is |
| 2312 | /// broadcast over two lane groups onto the rows of a subset of the lanes. |
| 2313 | /// Column `c` of row `r` is owned by lane `r + c * stride`, where `stride` is |
| 2314 | /// the lane stride of the input's distributed dimension (see |
| 2315 | /// `getDistributedDimLaneStride`), so each lane extracts its own element and |
| 2316 | /// gathers the columns of its row with one `gpu.shuffle idx` per column. |
| 2317 | /// |
| 2318 | /// The input layout has effective `lane_layout` [1, 2], so the distributed |
| 2319 | /// source holds one column per lane group; the target has [n, 1], so lane `l` |
| 2320 | /// keeps row `l % n` of both columns and the distributed result is |
| 2321 | /// `vector<1x2>`. |
| 2322 | /// |
| 2323 | /// xegpu.convert_layout %src |
| 2324 | /// <{input_layout = #xegpu.slice<#xegpu.layout<lane_layout = [8, 1, 2], |
| 2325 | /// lane_data = [4, 1, 1], |
| 2326 | /// order = [0, 2, 1]>, |
| 2327 | /// dims = [0]>, |
| 2328 | /// target_layout = #xegpu.layout<lane_layout = [8, 1], |
| 2329 | /// lane_data = [1, 1]>}> |
| 2330 | /// : vector<8x2xf8E8M0FNU> |
| 2331 | /// |
| 2332 | /// becomes, with lane `l` holding all 8 rows of column `l / 8`: |
| 2333 | /// |
| 2334 | /// %flat = vector.shape_cast %src : vector<8x1xf8E8M0FNU> to |
| 2335 | /// vector<8xf8E8M0FNU> |
| 2336 | /// %lane = gpu.lane_id |
| 2337 | /// %row = arith.remui %lane, %c8 : index |
| 2338 | /// %own = vector.extract %flat[%row] : f8E8M0FNU from |
| 2339 | /// vector<8xf8E8M0FNU> |
| 2340 | /// %rowI32 = arith.index_cast %row : index to i32 |
| 2341 | /// %owner1 = arith.addi %rowI32, %c8_i32 : i32 |
| 2342 | /// %col0, %v0 = gpu.shuffle idx %own, %rowI32, %c16_i32 : f8E8M0FNU |
| 2343 | /// %col1, %v1 = gpu.shuffle idx %own, %owner1, %c16_i32 : f8E8M0FNU |
| 2344 | /// %res = vector.from_elements %col0, %col1 : vector<1x2xf8E8M0FNU> |
| 2345 | /// |
| 2346 | /// The source is flattened first because `xegpu-vector-linearize` cannot |
| 2347 | /// linearize a `vector.extract` with a dynamic position out of a rank-2 value. |
| 2348 | /// |
| 2349 | /// All lanes gather every column, including the one they already hold, so that |
| 2350 | /// lanes outside the target `lane_layout` hold replicas as partial lane layouts |
| 2351 | /// require. |
| 2352 | struct |
| 2353 | : public OpConversionPattern<xegpu::ConvertLayoutOp> { |
| 2354 | using OpConversionPattern<xegpu::ConvertLayoutOp>::OpConversionPattern; |
| 2355 | |
| 2356 | LogicalResult |
| 2357 | matchAndRewrite(xegpu::ConvertLayoutOp op, OpAdaptor adaptor, |
| 2358 | ConversionPatternRewriter &rewriter) const override { |
| 2359 | xegpu::DistributeLayoutAttr inputLayout = op.getEffectiveInputLayout(); |
| 2360 | xegpu::DistributeLayoutAttr targetLayout = op.getTargetLayoutAttr(); |
| 2361 | |
| 2362 | auto inputSlice = dyn_cast<xegpu::SliceAttr>(Val&: inputLayout); |
| 2363 | if (!inputSlice) |
| 2364 | return rewriter.notifyMatchFailure(arg&: op, |
| 2365 | msg: "input_layout must be #xegpu.slice" ); |
| 2366 | if (!isa<xegpu::LayoutAttr>(Val: targetLayout)) |
| 2367 | return rewriter.notifyMatchFailure(arg&: op, |
| 2368 | msg: "target_layout must be #xegpu.layout" ); |
| 2369 | |
| 2370 | FailureOr<ElementLaneRedistribution> redistribution = |
| 2371 | matchElementLaneRedistribution(op, rewriter); |
| 2372 | if (failed(Result: redistribution)) |
| 2373 | return failure(); |
| 2374 | |
| 2375 | VectorType valueType = redistribution->valueType; |
| 2376 | Type elementType = valueType.getElementType(); |
| 2377 | if (!elementType.isIntOrFloat() || |
| 2378 | !llvm::is_contained(Set: {8u, 16u, 32u, 64u}, |
| 2379 | Element: elementType.getIntOrFloatBitWidth())) |
| 2380 | return rewriter.notifyMatchFailure( |
| 2381 | arg&: op, msg: "element type must be an int or float of bit width 8, 16, 32 or " |
| 2382 | "64 to be carried by gpu.shuffle" ); |
| 2383 | |
| 2384 | if (redistribution->inputLaneLayout != SmallVector<int64_t>{1, 2}) |
| 2385 | return rewriter.notifyMatchFailure( |
| 2386 | arg&: op, msg: "input_layout effective lane_layout must be [1, 2]" ); |
| 2387 | SmallVector<int64_t> targetLaneLayout = redistribution->targetLaneLayout; |
| 2388 | if (targetLaneLayout[0] <= 1 || targetLaneLayout[1] != 1) |
| 2389 | return rewriter.notifyMatchFailure( |
| 2390 | arg&: op, msg: "target_layout effective lane_layout must be [n, 1] with n > 1" ); |
| 2391 | if (redistribution->distributedInput.getShape() != |
| 2392 | ArrayRef<int64_t>{valueType.getDimSize(idx: 0), 1}) |
| 2393 | return rewriter.notifyMatchFailure( |
| 2394 | arg&: op, msg: "distributed input_layout type must be vector<shape[0]x1>" ); |
| 2395 | if (redistribution->distributedTarget.getShape() != ArrayRef<int64_t>{1, 2}) |
| 2396 | return rewriter.notifyMatchFailure( |
| 2397 | arg&: op, msg: "distributed target_layout type must be vector<1x2>" ); |
| 2398 | |
| 2399 | FailureOr<int64_t> stride = getDistributedDimLaneStride(slice: inputSlice); |
| 2400 | if (failed(Result: stride)) |
| 2401 | return rewriter.notifyMatchFailure( |
| 2402 | arg&: op, msg: "input_layout parent must have exactly one non-sliced dimension " |
| 2403 | "with lane_layout extent greater than one" ); |
| 2404 | // The two broadcast groups must together cover the whole subgroup, so that |
| 2405 | // lane `r + c * stride` is the lane holding column `c` of row `r`. |
| 2406 | if (*stride * redistribution->inputLaneLayout[1] != |
| 2407 | redistribution->subgroupSize) |
| 2408 | return rewriter.notifyMatchFailure( |
| 2409 | arg&: op, msg: "input_layout distributed dimension lane stride times its extent " |
| 2410 | "must equal the subgroup size" ); |
| 2411 | // Lane `r + c * stride` must hold row `r`, i.e. (r + stride) % |
| 2412 | // lane_layout[0] must be r, which requires the row count to divide the |
| 2413 | // stride. |
| 2414 | if (*stride % targetLaneLayout[0] != 0) |
| 2415 | return rewriter.notifyMatchFailure( |
| 2416 | arg&: op, msg: "target_layout effective lane_layout[0] must divide the " |
| 2417 | "input_layout distributed dimension lane stride" ); |
| 2418 | |
| 2419 | Location loc = op.getLoc(); |
| 2420 | Value src = |
| 2421 | castValueTo(rewriter, v: cast<TypedValue<VectorType>>(Val: adaptor.getSource()), |
| 2422 | expectedTy: redistribution->distributedInput); |
| 2423 | auto flatType = |
| 2424 | VectorType::get(shape: {redistribution->distributedInput.getNumElements()}, |
| 2425 | elementType: valueType.getElementType()); |
| 2426 | Value flat = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: flatType, source: src); |
| 2427 | Value laneId = gpu::LaneIdOp::create(builder&: rewriter, location: loc, result: rewriter.getIndexType(), |
| 2428 | /*upperBound=*/upper_bound: mlir::IntegerAttr()); |
| 2429 | Value rowCount = |
| 2430 | arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: targetLaneLayout[0]); |
| 2431 | Value row = arith::RemUIOp::create(builder&: rewriter, location: loc, lhs: laneId, rhs: rowCount); |
| 2432 | Value own = vector::ExtractOp::create(builder&: rewriter, location: loc, source: flat, |
| 2433 | position: ArrayRef<OpFoldResult>{row}); |
| 2434 | // Gather column `c` of `row` from lane `row + c * stride`, which owns it. |
| 2435 | Type i32Type = rewriter.getI32Type(); |
| 2436 | Value width = arith::ConstantIntOp::create(builder&: rewriter, location: loc, type: i32Type, |
| 2437 | value: redistribution->subgroupSize); |
| 2438 | Value rowI32 = arith::IndexCastOp::create(builder&: rewriter, location: loc, out: i32Type, in: row); |
| 2439 | Value strideI32 = |
| 2440 | arith::ConstantIntOp::create(builder&: rewriter, location: loc, type: i32Type, value: *stride); |
| 2441 | Value otherOwner = arith::AddIOp::create(builder&: rewriter, location: loc, lhs: rowI32, rhs: strideI32); |
| 2442 | Value column0 = gpu::ShuffleOp::create(builder&: rewriter, location: loc, value: own, offset: rowI32, width, |
| 2443 | mode: gpu::ShuffleMode::IDX) |
| 2444 | .getShuffleResult(); |
| 2445 | Value column1 = gpu::ShuffleOp::create(builder&: rewriter, location: loc, value: own, offset: otherOwner, |
| 2446 | width, mode: gpu::ShuffleMode::IDX) |
| 2447 | .getShuffleResult(); |
| 2448 | rewriter.replaceOpWithNewOp<vector::FromElementsOp>( |
| 2449 | op, args&: redistribution->distributedTarget, args: ValueRange{column0, column1}); |
| 2450 | return success(); |
| 2451 | } |
| 2452 | }; |
| 2453 | |
| 2454 | /// Distributes the slice-attributed `xegpu.convert_layout` whose source is |
| 2455 | /// fully broadcast and whose target splits the two columns of the value over |
| 2456 | /// two lane groups. Each lane keeps one whole column, which is a stride-2 |
| 2457 | /// subset of the row-major source, so the two columns are separated with one |
| 2458 | /// `vector.deinterleave` and the lane's group selects between them. |
| 2459 | /// |
| 2460 | /// The input layout has effective `lane_layout` [1, 1], so the distributed |
| 2461 | /// source is the whole value; the target has [1, 2] and its two groups tile the |
| 2462 | /// subgroup, so the first half of the lanes keeps column 0 and the second half |
| 2463 | /// column 1, one column per lane. |
| 2464 | struct SgToLaneConvertLayoutDeinterleaveSelect |
| 2465 | : public OpConversionPattern<xegpu::ConvertLayoutOp> { |
| 2466 | using OpConversionPattern<xegpu::ConvertLayoutOp>::OpConversionPattern; |
| 2467 | |
| 2468 | LogicalResult |
| 2469 | matchAndRewrite(xegpu::ConvertLayoutOp op, OpAdaptor adaptor, |
| 2470 | ConversionPatternRewriter &rewriter) const override { |
| 2471 | xegpu::DistributeLayoutAttr inputLayout = op.getEffectiveInputLayout(); |
| 2472 | xegpu::DistributeLayoutAttr targetLayout = op.getTargetLayoutAttr(); |
| 2473 | |
| 2474 | if (!isa<xegpu::SliceAttr>(Val: inputLayout)) |
| 2475 | return rewriter.notifyMatchFailure(arg&: op, |
| 2476 | msg: "input_layout must be #xegpu.slice" ); |
| 2477 | auto targetSlice = dyn_cast<xegpu::SliceAttr>(Val&: targetLayout); |
| 2478 | if (!targetSlice) |
| 2479 | return rewriter.notifyMatchFailure(arg&: op, |
| 2480 | msg: "target_layout must be #xegpu.slice" ); |
| 2481 | |
| 2482 | FailureOr<ElementLaneRedistribution> redistribution = |
| 2483 | matchElementLaneRedistribution(op, rewriter); |
| 2484 | if (failed(Result: redistribution)) |
| 2485 | return failure(); |
| 2486 | |
| 2487 | VectorType valueType = redistribution->valueType; |
| 2488 | if (redistribution->inputLaneLayout != SmallVector<int64_t>{1, 1}) |
| 2489 | return rewriter.notifyMatchFailure( |
| 2490 | arg&: op, msg: "input_layout effective lane_layout must be [1, 1]" ); |
| 2491 | SmallVector<int64_t> targetLaneLayout = redistribution->targetLaneLayout; |
| 2492 | if (targetLaneLayout != SmallVector<int64_t>{1, 2}) |
| 2493 | return rewriter.notifyMatchFailure( |
| 2494 | arg&: op, msg: "target_layout effective lane_layout must be [1, 2]" ); |
| 2495 | if (redistribution->distributedInput != valueType) |
| 2496 | return rewriter.notifyMatchFailure( |
| 2497 | arg&: op, msg: "distributed input_layout type must equal the value type" ); |
| 2498 | if (redistribution->distributedTarget.getShape() != |
| 2499 | ArrayRef<int64_t>{valueType.getDimSize(idx: 0), 1}) |
| 2500 | return rewriter.notifyMatchFailure( |
| 2501 | arg&: op, msg: "distributed target_layout type must be vector<shape[0]x1>" ); |
| 2502 | |
| 2503 | FailureOr<int64_t> stride = getDistributedDimLaneStride(slice: targetSlice); |
| 2504 | if (failed(Result: stride)) |
| 2505 | return rewriter.notifyMatchFailure( |
| 2506 | arg&: op, msg: "target_layout parent must have exactly one non-sliced dimension " |
| 2507 | "with lane_layout extent greater than one" ); |
| 2508 | // The two lane groups must together cover the whole subgroup, so that |
| 2509 | // `lane_id / stride` is the index of the column the lane keeps. |
| 2510 | if (*stride * targetLaneLayout[1] != redistribution->subgroupSize) |
| 2511 | return rewriter.notifyMatchFailure( |
| 2512 | arg&: op, msg: "target_layout distributed dimension lane stride times its " |
| 2513 | "extent must equal the subgroup size" ); |
| 2514 | |
| 2515 | Location loc = op.getLoc(); |
| 2516 | Value src = |
| 2517 | castValueTo(rewriter, v: cast<TypedValue<VectorType>>(Val: adaptor.getSource()), |
| 2518 | expectedTy: redistribution->distributedInput); |
| 2519 | auto flatType = VectorType::get(shape: {valueType.getNumElements()}, |
| 2520 | elementType: valueType.getElementType()); |
| 2521 | Value flat = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: flatType, source: src); |
| 2522 | auto deinterleaved = vector::DeinterleaveOp::create(builder&: rewriter, location: loc, source: flat); |
| 2523 | Value laneId = gpu::LaneIdOp::create(builder&: rewriter, location: loc, result: rewriter.getIndexType(), |
| 2524 | /*upperBound=*/upper_bound: mlir::IntegerAttr()); |
| 2525 | Value strideVal = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: *stride); |
| 2526 | Value zero = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0); |
| 2527 | Value group = arith::DivUIOp::create(builder&: rewriter, location: loc, lhs: laneId, rhs: strideVal); |
| 2528 | Value isFirstGroup = arith::CmpIOp::create( |
| 2529 | builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::eq, lhs: group, rhs: zero); |
| 2530 | Value selected = arith::SelectOp::create(builder&: rewriter, location: loc, condition: isFirstGroup, |
| 2531 | true_value: deinterleaved.getResultEven(), |
| 2532 | false_value: deinterleaved.getResultOdd()); |
| 2533 | rewriter.replaceOpWithNewOp<vector::ShapeCastOp>( |
| 2534 | op, args&: redistribution->distributedTarget, args&: selected); |
| 2535 | return success(); |
| 2536 | } |
| 2537 | }; |
| 2538 | |
| 2539 | // Trivially distribute `vector.interleave` |
| 2540 | struct SgToLaneVectorInterleave |
| 2541 | : public OpConversionPattern<vector::InterleaveOp> { |
| 2542 | using OpConversionPattern<vector::InterleaveOp>::OpConversionPattern; |
| 2543 | |
| 2544 | LogicalResult |
| 2545 | matchAndRewrite(vector::InterleaveOp op, OpAdaptor adaptor, |
| 2546 | ConversionPatternRewriter &rewriter) const override { |
| 2547 | |
| 2548 | auto newOp = vector::InterleaveOp::create( |
| 2549 | builder&: rewriter, location: op.getLoc(), lhs: adaptor.getLhs(), rhs: adaptor.getRhs()); |
| 2550 | rewriter.replaceOp(op, newValues: newOp.getResult()); |
| 2551 | return success(); |
| 2552 | } |
| 2553 | }; |
| 2554 | |
| 2555 | // Trivially distribute `vector.deinterleave` |
| 2556 | struct SgToLaneVectorDeinterleave |
| 2557 | : public OpConversionPattern<vector::DeinterleaveOp> { |
| 2558 | using OpConversionPattern<vector::DeinterleaveOp>::OpConversionPattern; |
| 2559 | |
| 2560 | LogicalResult |
| 2561 | matchAndRewrite(vector::DeinterleaveOp op, OpAdaptor adaptor, |
| 2562 | ConversionPatternRewriter &rewriter) const override { |
| 2563 | |
| 2564 | auto newOp = vector::DeinterleaveOp::create(builder&: rewriter, location: op.getLoc(), |
| 2565 | source: adaptor.getSource()); |
| 2566 | rewriter.replaceOp(op, newValues: newOp.getResults()); |
| 2567 | return success(); |
| 2568 | } |
| 2569 | }; |
| 2570 | |
| 2571 | struct SgToLaneDpasMx : public OpConversionPattern<xegpu::DpasMxOp> { |
| 2572 | using OpConversionPattern<xegpu::DpasMxOp>::OpConversionPattern; |
| 2573 | |
| 2574 | LogicalResult |
| 2575 | matchAndRewrite(xegpu::DpasMxOp op, OpAdaptor adaptor, |
| 2576 | ConversionPatternRewriter &rewriter) const override { |
| 2577 | const auto *uArch = |
| 2578 | xegpu::uArch::getUArch(archName: xegpu::getChipStr(op).value_or(u: "" )); |
| 2579 | if (!uArch) |
| 2580 | return failure(); |
| 2581 | if (!uArch->isSupportedInstruction( |
| 2582 | instr: xegpu::uArch::InstructionKind::SubgroupScaledMatrixMultiplyAcc)) |
| 2583 | return rewriter.notifyMatchFailure( |
| 2584 | arg&: op, msg: "target uArch does not support scaled subgroup mma" ); |
| 2585 | // Check if the op has A, B and CD layouts attached. |
| 2586 | auto layoutA = cast<xegpu::LayoutAttr>(Val: op.getLayoutAAttr()); |
| 2587 | auto layoutB = cast<xegpu::LayoutAttr>(Val: op.getLayoutBAttr()); |
| 2588 | auto layoutCd = cast<xegpu::LayoutAttr>(Val: op.getLayoutCdAttr()); |
| 2589 | if (!layoutA || !layoutB || !layoutCd) |
| 2590 | return rewriter.notifyMatchFailure( |
| 2591 | arg&: op, msg: "missing required layout attributes for DpasMxOp distribution" ); |
| 2592 | |
| 2593 | // Retrieve expected types, according to anchor layouts. |
| 2594 | auto expected1DTypeResult = |
| 2595 | xegpu::getDistributedVectorType(originalType: op.getType(), layout: layoutCd); |
| 2596 | auto expected1DTypeA = |
| 2597 | xegpu::getDistributedVectorType(originalType: op.getA().getType(), layout: layoutA); |
| 2598 | auto expected1DTypeB = |
| 2599 | xegpu::getDistributedVectorType(originalType: op.getB().getType(), layout: layoutB); |
| 2600 | |
| 2601 | VectorType expected1DTypeScaleA, expected1DTypeScaleB; |
| 2602 | if (op.getScaleA()) { |
| 2603 | auto layoutScaleA = cast<xegpu::LayoutAttr>(Val: op.getLayoutAScaleAttr()); |
| 2604 | auto expected1DTypeScaleAOrFailure = xegpu::getDistributedVectorType( |
| 2605 | originalType: cast<VectorType>(Val: op.getScaleA().getType()), layout: layoutScaleA); |
| 2606 | if (failed(Result: expected1DTypeScaleAOrFailure)) |
| 2607 | return rewriter.notifyMatchFailure( |
| 2608 | arg&: op, msg: "failed to calculate expected 1D vector type for scale A" ); |
| 2609 | expected1DTypeScaleA = expected1DTypeScaleAOrFailure.value(); |
| 2610 | } |
| 2611 | if (op.getScaleB()) { |
| 2612 | auto layoutScaleB = cast<xegpu::LayoutAttr>(Val: op.getLayoutBScaleAttr()); |
| 2613 | auto expected1DTypeScaleBOrFailure = xegpu::getDistributedVectorType( |
| 2614 | originalType: cast<VectorType>(Val: op.getScaleB().getType()), layout: layoutScaleB); |
| 2615 | if (failed(Result: expected1DTypeScaleBOrFailure)) |
| 2616 | return rewriter.notifyMatchFailure( |
| 2617 | arg&: op, msg: "failed to calculate expected 1D vector type for scale B" ); |
| 2618 | expected1DTypeScaleB = expected1DTypeScaleBOrFailure.value(); |
| 2619 | } |
| 2620 | |
| 2621 | auto expectedNDTypeResult = |
| 2622 | xegpu::getDistVecTypeBasedOnLaneLayout(layout: layoutCd, originalType: op.getType()); |
| 2623 | if (failed(Result: expected1DTypeResult) || failed(Result: expected1DTypeA) || |
| 2624 | failed(Result: expected1DTypeB)) |
| 2625 | return rewriter.notifyMatchFailure( |
| 2626 | arg&: op, |
| 2627 | msg: "failed to calculate supported workitem 1D vector types for DpasOp " |
| 2628 | "from layouts" ); |
| 2629 | if (failed(Result: expectedNDTypeResult)) |
| 2630 | return rewriter.notifyMatchFailure( |
| 2631 | arg&: op, msg: "unable to compute expected workitem vector type for DpasOp from " |
| 2632 | "lane layout" ); |
| 2633 | |
| 2634 | // Validate bit widths match uArch packed format requirements |
| 2635 | const auto *uArchInstruction = dyn_cast< |
| 2636 | xegpu::uArch::SubgroupScaledMatrixMultiplyAcc>(Val: uArch->getInstruction( |
| 2637 | instKind: xegpu::uArch::InstructionKind::SubgroupScaledMatrixMultiplyAcc)); |
| 2638 | assert(uArchInstruction); |
| 2639 | auto wiAType = expected1DTypeA.value(); |
| 2640 | auto wiBType = expected1DTypeB.value(); |
| 2641 | // Calculate total packed bit width = element bit width * vector size |
| 2642 | unsigned aPackedBitWidth = |
| 2643 | wiAType.getElementTypeBitWidth() * wiAType.getNumElements(); |
| 2644 | unsigned bPackedBitWidth = |
| 2645 | wiBType.getElementTypeBitWidth() * wiBType.getNumElements(); |
| 2646 | if (aPackedBitWidth % uArchInstruction->getPackedFormatBitSizeA()) |
| 2647 | return rewriter.notifyMatchFailure( |
| 2648 | arg&: op, msg: "A operand packed bit width must be a multiple of uArch packed " |
| 2649 | "format requirement" ); |
| 2650 | if (bPackedBitWidth % uArchInstruction->getPackedFormatBitSizeB()) |
| 2651 | return rewriter.notifyMatchFailure( |
| 2652 | arg&: op, msg: "B operand packed bit width must be a multiple of uArch packed " |
| 2653 | "format requirement" ); |
| 2654 | |
| 2655 | auto newOp = xegpu::DpasMxOp::create( |
| 2656 | builder&: rewriter, location: op->getLoc(), result: expected1DTypeResult.value(), |
| 2657 | a: castValueTo(rewriter, v: cast<TypedValue<VectorType>>(Val: adaptor.getA()), |
| 2658 | expectedTy: expected1DTypeA.value()), |
| 2659 | b: castValueTo(rewriter, v: cast<TypedValue<VectorType>>(Val: adaptor.getB()), |
| 2660 | expectedTy: expected1DTypeB.value()), |
| 2661 | acc: op.getAcc() |
| 2662 | ? castValueTo(rewriter, |
| 2663 | v: cast<TypedValue<VectorType>>(Val: adaptor.getAcc()), |
| 2664 | expectedTy: expected1DTypeResult.value()) |
| 2665 | : nullptr, |
| 2666 | |
| 2667 | scale_a: op.getScaleA() |
| 2668 | ? castValueTo(rewriter, |
| 2669 | v: cast<TypedValue<VectorType>>(Val: adaptor.getScaleA()), |
| 2670 | expectedTy: expected1DTypeScaleA) |
| 2671 | : nullptr, |
| 2672 | scale_b: op.getScaleB() |
| 2673 | ? castValueTo(rewriter, |
| 2674 | v: cast<TypedValue<VectorType>>(Val: adaptor.getScaleB()), |
| 2675 | expectedTy: expected1DTypeScaleB) |
| 2676 | : nullptr, |
| 2677 | /** layoutA**/ layout_a: nullptr, |
| 2678 | /** layoutB**/ layout_b: nullptr, /** layoutCd**/ layout_cd: nullptr, |
| 2679 | /** layoutAScale**/ layout_a_scale: nullptr, /** layoutBScale**/ layout_b_scale: nullptr); |
| 2680 | // Explicitly set the new types to enable correct type materializations. |
| 2681 | rewriter.replaceOp(op, newValues: castValueTo(rewriter, v: newOp.getResult(), |
| 2682 | expectedTy: expectedNDTypeResult.value())); |
| 2683 | return success(); |
| 2684 | } |
| 2685 | }; |
| 2686 | |
| 2687 | struct XeGPUSgToLaneDistributePass |
| 2688 | : public xegpu::impl::XeGPUSgToLaneDistributeBase< |
| 2689 | XeGPUSgToLaneDistributePass> { |
| 2690 | void runOnOperation() override; |
| 2691 | }; |
| 2692 | |
| 2693 | } // namespace |
| 2694 | |
| 2695 | void XeGPUSgToLaneDistributePass::runOnOperation() { |
| 2696 | |
| 2697 | // Recover temporary operand layouts for usage in patterns. |
| 2698 | Operation *root = getOperation(); |
| 2699 | if (!xegpu::recoverTemporaryLayouts(rootOp: root)) { |
| 2700 | signalPassFailure(); |
| 2701 | return; |
| 2702 | } |
| 2703 | |
| 2704 | // Collect existing UnrealizedConversionCastOps. These must be preserved. |
| 2705 | llvm::SmallSetVector<UnrealizedConversionCastOp, 8> existingCasts; |
| 2706 | root->walk( |
| 2707 | callback: [&](UnrealizedConversionCastOp castOp) { existingCasts.insert(X: castOp); }); |
| 2708 | // Perform a structural type conversion to convert structural ops to have WI |
| 2709 | // types. This will insert UnrealizedConversionCastOps to make the IR |
| 2710 | // valid. |
| 2711 | { |
| 2712 | ConversionTarget target(getContext()); |
| 2713 | TypeConverter typeConverter; |
| 2714 | RewritePatternSet patterns(&getContext()); |
| 2715 | // Source (N:1) and target (1:1) materializations using |
| 2716 | // UnrealizedConversionCastOp. |
| 2717 | auto materializeCast = [](OpBuilder &builder, Type type, ValueRange inputs, |
| 2718 | Location loc) -> Value { |
| 2719 | return UnrealizedConversionCastOp::create(builder, location: loc, resultTypes: type, operands: inputs) |
| 2720 | .getResult(i: 0); |
| 2721 | }; |
| 2722 | typeConverter.addSourceMaterialization(callback&: materializeCast); |
| 2723 | typeConverter.addTargetMaterialization(callback&: materializeCast); |
| 2724 | xegpu::populateXeGPUSgToLaneDistributeTypeConversions(typeConverter, topLevelOp: root); |
| 2725 | scf::populateSCFStructuralTypeConversionsAndLegality(typeConverter, |
| 2726 | patterns, target); |
| 2727 | xegpu::populateXeGPUSgToLaneDistributeTypeConversionAndLegality( |
| 2728 | typeConverter, patterns, target, topLevelOp: root); |
| 2729 | target.addLegalOp<UnrealizedConversionCastOp>(); |
| 2730 | // Fold only after the patterns. A fold builds new ops without the |
| 2731 | // temporary layout attribute, so they are never distributed. For example, |
| 2732 | // an all-ones vector.constant_mask folds to an arith.constant that stays |
| 2733 | // at subgroup size behind an unrealized cast. |
| 2734 | ConversionConfig config; |
| 2735 | config.foldingMode = DialectConversionFoldingMode::AfterPatterns; |
| 2736 | if (failed( |
| 2737 | Result: applyPartialConversion(op: root, target, patterns: std::move(patterns), config))) |
| 2738 | return signalPassFailure(); |
| 2739 | } |
| 2740 | // Fold cancelling cast chains and erase dead casts. |
| 2741 | xegpu::cleanupUnrealizedConversionCasts(root, existingCasts); |
| 2742 | xegpu::removeTemporaryLayoutAttrs(op: getOperation()); |
| 2743 | } |
| 2744 | |
| 2745 | void xegpu::populateXeGPUSgToLaneDistributeTypeConversions( |
| 2746 | TypeConverter &typeConverter, Operation *topLevelOp) { |
| 2747 | // Pass through any type by default; more specific conversions registered |
| 2748 | // below override this for TensorDescType and (distributing) VectorType. |
| 2749 | typeConverter.addConversion(callback: [](Type type) -> Type { return type; }); |
| 2750 | // For TensorDescType, drop the layout attribute if any. |
| 2751 | typeConverter.addConversion(callback: [](TensorDescType type) -> Type { |
| 2752 | if (type.getLayoutAttr()) { |
| 2753 | return type.dropLayouts(); |
| 2754 | } |
| 2755 | return type; |
| 2756 | }); |
| 2757 | // For VectorType, distribute based on the lane layout (1:1 shape-changing |
| 2758 | // conversion). Uses xegpu::addVectorTypeConversion with a pre-computed |
| 2759 | // map for SCF loop block args (see precomputeLoopBlockArgTypes for the |
| 2760 | // rationale). |
| 2761 | auto getSubShapeAndCount = [](VectorType vecTy, |
| 2762 | xegpu::DistributeLayoutAttr layout) |
| 2763 | -> std::pair<SmallVector<int64_t>, int> { |
| 2764 | auto distTyOrFailure = getDistVecTypeBasedOnLaneLayout(layout, originalType: vecTy); |
| 2765 | if (failed(Result: distTyOrFailure)) |
| 2766 | return {{}, 0}; |
| 2767 | return {SmallVector<int64_t>(distTyOrFailure->getShape()), 1}; |
| 2768 | }; |
| 2769 | auto loopArgTypes = |
| 2770 | xegpu::precomputeLoopBlockArgTypes(topLevelOp, getSubShapeAndCount); |
| 2771 | xegpu::addVectorTypeConversion(converter&: typeConverter, getSubShapeAndCount, |
| 2772 | loopArgTypes: std::move(loopArgTypes)); |
| 2773 | } |
| 2774 | |
| 2775 | void xegpu::populateXeGPUSgToLaneDistributeTypeConversionAndLegality( |
| 2776 | TypeConverter &typeConverter, RewritePatternSet &patterns, |
| 2777 | ConversionTarget &target, Operation *topLevelOp) { |
| 2778 | populateXeGPUSgToLaneDistributeTypeConversions(typeConverter, topLevelOp); |
| 2779 | // CreateNdDescOp is legal only if its result type has no layout attribute. |
| 2780 | target.addDynamicallyLegalOp<xegpu::CreateNdDescOp>( |
| 2781 | callback: [&](xegpu::CreateNdDescOp op) { return !op.getType().getLayoutAttr(); }); |
| 2782 | // Any anchor XeGPU op is legal only if it has no anchor layout. |
| 2783 | target.addDynamicallyLegalDialect<xegpu::XeGPUDialect>(callback: [](Operation *op) { |
| 2784 | if (isa<xegpu::ConvertLayoutOp>(Val: op)) |
| 2785 | return false; |
| 2786 | auto anchorOp = dyn_cast<AnchorLayoutInterface>(Val: op); |
| 2787 | if (!anchorOp) |
| 2788 | return true; |
| 2789 | return !anchorOp.getAnchorLayout(); |
| 2790 | }); |
| 2791 | // Arith constants are legal only if they have no temporary layout attribute. |
| 2792 | target.addDynamicallyLegalOp<arith::ConstantOp>( |
| 2793 | callback: [=](arith::ConstantOp op) -> bool { |
| 2794 | // If the result type is not a vector, it's legal. |
| 2795 | if (!isa<VectorType>(Val: op.getResult().getType())) |
| 2796 | return true; |
| 2797 | return !xegpu::getTemporaryLayout(operandOrResult: dyn_cast<OpResult>(Val: op.getResult())); |
| 2798 | }); |
| 2799 | // In math and arith dialects, only handle elementwise ops with a single |
| 2800 | // result and with a result layout attribute. |
| 2801 | target.addDynamicallyLegalDialect<math::MathDialect, arith::ArithDialect>( |
| 2802 | callback: [=](Operation *op) -> std::optional<bool> { |
| 2803 | // Only handle elementwise mappable ops |
| 2804 | if (!OpTrait::hasElementwiseMappableTraits(op)) |
| 2805 | return true; |
| 2806 | // Only handle ops with single vector result |
| 2807 | if (op->getNumResults() != 1) |
| 2808 | return true; |
| 2809 | |
| 2810 | VectorType resultType = |
| 2811 | dyn_cast<VectorType>(Val: op->getResult(idx: 0).getType()); |
| 2812 | if (!resultType) |
| 2813 | return true; |
| 2814 | |
| 2815 | // Check if all operands are vectors of the same shape |
| 2816 | for (Value operand : op->getOperands()) { |
| 2817 | VectorType operandType = dyn_cast<VectorType>(Val: operand.getType()); |
| 2818 | if (!operandType || operandType.getShape() != resultType.getShape()) { |
| 2819 | return true; |
| 2820 | } |
| 2821 | } |
| 2822 | return !xegpu::getTemporaryLayout(operandOrResult: dyn_cast<OpResult>(Val: op->getResult(idx: 0))); |
| 2823 | }); |
| 2824 | // vector::ReductionOp is legal only if its source has no distribute layout |
| 2825 | // attribute. |
| 2826 | target.addDynamicallyLegalOp<vector::ReductionOp>( |
| 2827 | callback: [=](vector::ReductionOp op) -> bool { |
| 2828 | auto layout = xegpu::getDistributeLayoutAttr(value: op.getVector()); |
| 2829 | return !layout; |
| 2830 | }); |
| 2831 | // vector::MultiDimReductionOp op legality. |
| 2832 | target.addDynamicallyLegalOp<vector::MultiDimReductionOp>( |
| 2833 | callback: [=](vector::MultiDimReductionOp op) -> bool { |
| 2834 | return !isValidSubgroupMultiReductionOp(op); |
| 2835 | }); |
| 2836 | target.addDynamicallyLegalOp<vector::CreateMaskOp, vector::ConstantMaskOp, |
| 2837 | vector::TransposeOp, vector::BitCastOp, |
| 2838 | vector::ShapeCastOp, vector::StepOp, |
| 2839 | vector::BroadcastOp>(callback: [=](Operation *op) -> bool { |
| 2840 | return !xegpu::getTemporaryLayout(operandOrResult: op->getOpResult(idx: 0)); |
| 2841 | }); |
| 2842 | target.addDynamicallyLegalOp<vector::ExtractOp>( |
| 2843 | callback: [=](vector::ExtractOp op) -> bool { |
| 2844 | if (!isa<VectorType>(Val: op.getType())) |
| 2845 | return true; |
| 2846 | return !xegpu::getTemporaryLayout(operandOrResult: op->getOpResult(idx: 0)); |
| 2847 | }); |
| 2848 | target.addDynamicallyLegalOp<vector::InsertOp>( |
| 2849 | callback: [=](vector::InsertOp op) -> bool { |
| 2850 | return !xegpu::getTemporaryLayout(operandOrResult: op->getOpResult(idx: 0)); |
| 2851 | }); |
| 2852 | target.addDynamicallyLegalOp<vector::ExtractStridedSliceOp>( |
| 2853 | callback: [=](vector::ExtractStridedSliceOp op) -> bool { |
| 2854 | return !xegpu::getTemporaryLayout(operandOrResult: op->getOpResult(idx: 0)); |
| 2855 | }); |
| 2856 | target.addDynamicallyLegalOp<vector::InsertStridedSliceOp>( |
| 2857 | callback: [=](vector::InsertStridedSliceOp op) -> bool { |
| 2858 | return !xegpu::getTemporaryLayout(operandOrResult: op->getOpResult(idx: 0)); |
| 2859 | }); |
| 2860 | target.addDynamicallyLegalOp<vector::InterleaveOp, vector::DeinterleaveOp>( |
| 2861 | callback: [=](Operation *op) -> bool { |
| 2862 | return !xegpu::getTemporaryLayout(operandOrResult: op->getOpResult(idx: 0)); |
| 2863 | }); |
| 2864 | target.markUnknownOpDynamicallyLegal(fn: [](Operation *op) { return true; }); |
| 2865 | patterns |
| 2866 | .add<SgToLaneCreateNdDesc, SgToLaneLoadNd, SgToLaneStoreNd, SgToLaneDpas, |
| 2867 | SgToLaneElementWise, SgToLaneArithConstant, SgToLanePrefetchNd, |
| 2868 | SgToLaneLoadGather, SgToLaneStoreScatter, SgToLaneVectorReduction, |
| 2869 | SgToLaneMultiDimReduction, SgToLaneVectorExtract, |
| 2870 | SgToLaneVectorInsert, SgToLaneVectorExtractStridedSlice, |
| 2871 | SgToLaneVectorInsertStridedSlice, SgToLaneLoadMatrix, |
| 2872 | SgToLaneStoreMatrix, SgToLaneConvertLayout, SgToLaneVectorTranspose, |
| 2873 | SgToLaneVectorBitcast, SgToLaneVectorStep, SgToLaneVectorShapeCast, |
| 2874 | SgToLaneBroadcast, SgToLaneCreateMask<vector::CreateMaskOp>, |
| 2875 | SgToLaneCreateMask<vector::ConstantMaskOp>, |
| 2876 | SgToLaneVectorDeinterleave, SgToLaneVectorInterleave, SgToLaneDpasMx, |
| 2877 | SgToLaneConvertLayoutBroadcastExtract, |
| 2878 | SgToLaneConvertLayoutPartialBroadcastExtractShuffle, |
| 2879 | SgToLaneConvertLayoutDeinterleaveSelect>(arg&: typeConverter, |
| 2880 | args: patterns.getContext()); |
| 2881 | // The SLM round-trip handles any pair of layouts but pays for a memory |
| 2882 | // round-trip, so it only runs when every pattern above has declined. |
| 2883 | patterns.add<SgToLaneConvertLayoutViaSLM>(arg&: typeConverter, |
| 2884 | args: patterns.getContext(), |
| 2885 | /*benefit=*/args: PatternBenefit(0)); |
| 2886 | } |
| 2887 | |