| 1 | //===- XeGPUTransformOps.cpp - Implementation of XeGPU transformation ops -===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Dialect/XeGPU/TransformOps/XeGPUTransformOps.h" |
| 10 | #include "mlir/Analysis/SliceAnalysis.h" |
| 11 | #include "mlir/Dialect/GPU/IR/GPUDialect.h" |
| 12 | #include "mlir/Dialect/SCF/IR/SCF.h" |
| 13 | #include "mlir/Dialect/SCF/Utils/Utils.h" |
| 14 | #include "mlir/Dialect/Transform/Utils/Utils.h" |
| 15 | #include "mlir/Dialect/XeGPU/IR/XeGPU.h" |
| 16 | #include "mlir/Dialect/XeGPU/Utils/XeGPUUtils.h" |
| 17 | #include "mlir/IR/Dominance.h" |
| 18 | #include "mlir/Interfaces/SideEffectInterfaces.h" |
| 19 | #include "llvm/ADT/SmallVectorExtras.h" |
| 20 | |
| 21 | #include <optional> |
| 22 | |
| 23 | #include "llvm/Support/DebugLog.h" |
| 24 | #define DEBUG_TYPE "xegpu-transforms" |
| 25 | |
| 26 | using namespace mlir; |
| 27 | using namespace mlir::transform; |
| 28 | |
| 29 | /// Assuming that `ofr` is an index attr or a param of index type |
| 30 | /// or a transform dialect handle mapped to exactly one op |
| 31 | /// with one index result, get that value and cast it to int type. |
| 32 | static DiagnosedSilenceableFailure convertMixedValuesToInt( |
| 33 | transform::TransformState &state, TransformOpInterface transformOp, |
| 34 | SmallVectorImpl<int32_t> &result, ArrayRef<OpFoldResult> ofrs) { |
| 35 | for (OpFoldResult ofr : ofrs) { |
| 36 | // Attribute case. |
| 37 | if (auto attr = dyn_cast<Attribute>(Val&: ofr)) { |
| 38 | if (auto intAttr = dyn_cast<IntegerAttr>(Val&: attr)) { |
| 39 | result.push_back(Elt: intAttr.getInt()); |
| 40 | continue; |
| 41 | } |
| 42 | return transformOp.emitDefiniteFailure() << "expected IntegerAttr" ; |
| 43 | } |
| 44 | |
| 45 | // Transform param case. |
| 46 | Value transformValue = cast<Value>(Val&: ofr); |
| 47 | if (isa<TransformParamTypeInterface>(Val: transformValue.getType())) { |
| 48 | ArrayRef<Attribute> params = state.getParams(value: transformValue); |
| 49 | if (params.size() != 1) |
| 50 | return transformOp.emitDefiniteFailure() |
| 51 | << "requires exactly one parameter associated" ; |
| 52 | result.push_back( |
| 53 | Elt: cast<IntegerAttr>(Val: params.front()).getValue().getSExtValue()); |
| 54 | continue; |
| 55 | } |
| 56 | |
| 57 | // Payload value case. |
| 58 | auto payloadOps = state.getPayloadOps(value: transformValue); |
| 59 | if (!llvm::hasSingleElement(C&: payloadOps)) { |
| 60 | DiagnosedSilenceableFailure diag = |
| 61 | transformOp.emitSilenceableError() |
| 62 | << "handle must be mapped to exactly one payload op" ; |
| 63 | diag.attachNote(loc: transformValue.getLoc()) |
| 64 | << "mapped to " << llvm::range_size(Range&: payloadOps) << " payload ops" ; |
| 65 | return diag; |
| 66 | } |
| 67 | |
| 68 | Operation *op = *payloadOps.begin(); |
| 69 | if (op->getNumResults() != 1 || !op->getResult(idx: 0).getType().isIndex()) { |
| 70 | DiagnosedSilenceableFailure diag = |
| 71 | transformOp.emitSilenceableError() |
| 72 | << "payload op must have exactly 1 index result" ; |
| 73 | diag.attachNote(loc: op->getLoc()) |
| 74 | << "has " << op->getNumResults() << " results" ; |
| 75 | return diag; |
| 76 | } |
| 77 | |
| 78 | IntegerAttr intAttr; |
| 79 | if (!matchPattern(value: op->getResult(idx: 0), pattern: m_Constant(bind_value: &intAttr))) |
| 80 | return transformOp.emitSilenceableError() |
| 81 | << "requires param or handle to be the result of a constant like " |
| 82 | "op" ; |
| 83 | |
| 84 | result.push_back(Elt: intAttr.getInt()); |
| 85 | } |
| 86 | return DiagnosedSilenceableFailure::success(); |
| 87 | } |
| 88 | |
| 89 | /// Convert a single packed handle, that is either a transform param associated |
| 90 | /// with a variadic number of integer attributes or mapped to a variadic number |
| 91 | /// of payload ops each defining a single constant index result, to a list of |
| 92 | /// int values. |
| 93 | static DiagnosedSilenceableFailure |
| 94 | convertPackedValueToInt(transform::TransformState &state, |
| 95 | TransformOpInterface transformOp, |
| 96 | SmallVectorImpl<int32_t> &result, Value packedHandle) { |
| 97 | // Transform param case. |
| 98 | if (isa<TransformParamTypeInterface>(Val: packedHandle.getType())) { |
| 99 | for (Attribute param : state.getParams(value: packedHandle)) { |
| 100 | auto intAttr = dyn_cast<IntegerAttr>(Val&: param); |
| 101 | if (!intAttr) |
| 102 | return transformOp.emitDefiniteFailure() |
| 103 | << "expected the parameter to be associated with an integer " |
| 104 | "attribute" ; |
| 105 | result.push_back(Elt: intAttr.getValue().getSExtValue()); |
| 106 | } |
| 107 | return DiagnosedSilenceableFailure::success(); |
| 108 | } |
| 109 | |
| 110 | // Payload op case. |
| 111 | for (Operation *op : state.getPayloadOps(value: packedHandle)) { |
| 112 | if (op->getNumResults() != 1 || !op->getResult(idx: 0).getType().isIndex()) { |
| 113 | DiagnosedSilenceableFailure diag = |
| 114 | transformOp.emitSilenceableError() |
| 115 | << "payload op must have exactly 1 index result" ; |
| 116 | diag.attachNote(loc: op->getLoc()) |
| 117 | << "has " << op->getNumResults() << " results" ; |
| 118 | return diag; |
| 119 | } |
| 120 | |
| 121 | IntegerAttr intAttr; |
| 122 | if (!matchPattern(value: op->getResult(idx: 0), pattern: m_Constant(bind_value: &intAttr))) |
| 123 | return transformOp.emitSilenceableError() |
| 124 | << "requires param or handle to be the result of a constant like " |
| 125 | "op" ; |
| 126 | |
| 127 | result.push_back(Elt: intAttr.getInt()); |
| 128 | } |
| 129 | return DiagnosedSilenceableFailure::success(); |
| 130 | } |
| 131 | |
| 132 | /// It's assumed that producer ops are chained through their first operand. |
| 133 | /// Producer chain is traced trough loop block arguments (init values). |
| 134 | template <typename T> |
| 135 | static std::optional<T> findProducerOfType(Value val) { |
| 136 | Value currentValue = val; |
| 137 | if (!currentValue.getDefiningOp()) { |
| 138 | // Value may be a block argument initialized outside a loop. |
| 139 | if (val.getNumUses() == 0) { |
| 140 | LDBG() << "Failed to find producer op, value has no uses." ; |
| 141 | return std::nullopt; |
| 142 | } |
| 143 | auto userOp = val.getUsers().begin(); |
| 144 | auto parentLoop = userOp->getParentOfType<LoopLikeOpInterface>(); |
| 145 | if (!parentLoop) { |
| 146 | LDBG() << "Failed to find producer op, not in a loop." ; |
| 147 | return std::nullopt; |
| 148 | } |
| 149 | int64_t iterArgIdx; |
| 150 | if (auto iterArg = llvm::dyn_cast<BlockArgument>(Val&: currentValue)) { |
| 151 | auto numInductionVars = parentLoop.getLoopInductionVars()->size(); |
| 152 | iterArgIdx = iterArg.getArgNumber() - numInductionVars; |
| 153 | currentValue = parentLoop.getInits()[iterArgIdx]; |
| 154 | } else { |
| 155 | LDBG() << "Failed to find producer op, value not in init values." ; |
| 156 | return std::nullopt; |
| 157 | } |
| 158 | } |
| 159 | Operation *producerOp = currentValue.getDefiningOp(); |
| 160 | |
| 161 | if (auto matchingOp = dyn_cast<T>(producerOp)) |
| 162 | return matchingOp; |
| 163 | |
| 164 | if (producerOp->getNumOperands() == 0) |
| 165 | return std::nullopt; |
| 166 | |
| 167 | return findProducerOfType<T>(producerOp->getOperand(idx: 0)); |
| 168 | } |
| 169 | |
| 170 | /// Create a layout attribute from the given parameters. |
| 171 | static xegpu::LayoutAttr createLayoutAttr( |
| 172 | MLIRContext *ctx, ArrayRef<int32_t> sgLayout, ArrayRef<int32_t> sgData, |
| 173 | std::optional<ArrayRef<int32_t>> instData, ArrayRef<int32_t> order) { |
| 174 | return xegpu::LayoutAttr::get( |
| 175 | context: ctx, sg_layout: DenseI32ArrayAttr::get(context: ctx, content: sgLayout), |
| 176 | sg_data: DenseI32ArrayAttr::get(context: ctx, content: sgData), |
| 177 | inst_data: instData ? DenseI32ArrayAttr::get(context: ctx, content: instData.value()) : nullptr, |
| 178 | /*lane_layout=*/nullptr, |
| 179 | /*lane_data=*/nullptr, |
| 180 | /*order=*/order.empty() ? nullptr : DenseI32ArrayAttr::get(context: ctx, content: order)); |
| 181 | } |
| 182 | |
| 183 | /// Generate `xegpu::LayoutAttr` from op mixed layout values. |
| 184 | DiagnosedSilenceableFailure |
| 185 | getLayoutAttrFromOperands(MLIRContext *ctx, transform::TransformState &state, |
| 186 | TransformOpInterface transformOp, |
| 187 | ArrayRef<::mlir::OpFoldResult> mixedSgLayout, |
| 188 | ArrayRef<::mlir::OpFoldResult> mixedSgData, |
| 189 | ArrayRef<::mlir::OpFoldResult> mixedInstData, |
| 190 | Value packedSgLayout, Value packedSgData, |
| 191 | Value packedInstData, ArrayRef<int32_t> order, |
| 192 | xegpu::LayoutAttr &layoutAttr) { |
| 193 | SmallVector<int32_t> sgLayout, sgData, instData; |
| 194 | |
| 195 | // A packed handle, when present, supersedes the mixed values and is unpacked |
| 196 | // to a variadic list of integers. |
| 197 | auto convert = [&](Value packed, ArrayRef<OpFoldResult> mixed, |
| 198 | SmallVectorImpl<int32_t> &out) { |
| 199 | return packed ? convertPackedValueToInt(state, transformOp, result&: out, packedHandle: packed) |
| 200 | : convertMixedValuesToInt(state, transformOp, result&: out, ofrs: mixed); |
| 201 | }; |
| 202 | |
| 203 | auto status = convert(packedSgLayout, mixedSgLayout, sgLayout); |
| 204 | if (!status.succeeded()) |
| 205 | return status; |
| 206 | |
| 207 | status = convert(packedSgData, mixedSgData, sgData); |
| 208 | if (!status.succeeded()) |
| 209 | return status; |
| 210 | |
| 211 | status = convert(packedInstData, mixedInstData, instData); |
| 212 | if (!status.succeeded()) |
| 213 | return status; |
| 214 | auto maybeInstData = instData.empty() |
| 215 | ? std::nullopt |
| 216 | : std::optional<ArrayRef<int32_t>>(instData); |
| 217 | |
| 218 | layoutAttr = createLayoutAttr(ctx, sgLayout, sgData, instData: maybeInstData, order); |
| 219 | |
| 220 | return DiagnosedSilenceableFailure::success(); |
| 221 | } |
| 222 | |
| 223 | DiagnosedSilenceableFailure |
| 224 | transform::GetLoadOp::apply(transform::TransformRewriter &rewriter, |
| 225 | transform::TransformResults &results, |
| 226 | transform::TransformState &state) { |
| 227 | auto targetValues = state.getPayloadValues(getTarget()); |
| 228 | if (!llvm::hasSingleElement(targetValues)) { |
| 229 | return emitDefiniteFailure() |
| 230 | << "requires exactly one target value handle (got " |
| 231 | << llvm::range_size(targetValues) << ")" ; |
| 232 | } |
| 233 | |
| 234 | Operation *loadOp = nullptr; |
| 235 | auto maybeLoadNdOp = |
| 236 | findProducerOfType<xegpu::LoadNdOp>(*targetValues.begin()); |
| 237 | if (maybeLoadNdOp) { |
| 238 | loadOp = maybeLoadNdOp->getOperation(); |
| 239 | } else { |
| 240 | auto maybeLoadOp = |
| 241 | findProducerOfType<xegpu::LoadGatherOp>(*targetValues.begin()); |
| 242 | if (maybeLoadOp) { |
| 243 | loadOp = maybeLoadOp->getOperation(); |
| 244 | } else { |
| 245 | return emitSilenceableFailure(getLoc()) |
| 246 | << "Could not find a matching xegpu.load_nd or xegpu.load op when " |
| 247 | "walking the " |
| 248 | "producer chain of the first operand." ; |
| 249 | } |
| 250 | } |
| 251 | |
| 252 | results.set(llvm::cast<OpResult>(getResult()), {loadOp}); |
| 253 | return DiagnosedSilenceableFailure::success(); |
| 254 | } |
| 255 | |
| 256 | void transform::SetAnchorLayoutOp::build( |
| 257 | OpBuilder &builder, OperationState &ostate, Value target, int64_t index, |
| 258 | ArrayRef<OpFoldResult> mixedSgLayout, ArrayRef<OpFoldResult> mixedSgData, |
| 259 | ArrayRef<OpFoldResult> mixedInstData, ArrayRef<int32_t> order, |
| 260 | ArrayRef<int64_t> sliceDims) { |
| 261 | SmallVector<int64_t> staticSgLayout, staticSgData, staticInstData; |
| 262 | SmallVector<Value> dynamicSgLayout, dynamicSgData, dynamicInstData; |
| 263 | dispatchIndexOpFoldResults(mixedSgLayout, dynamicSgLayout, staticSgLayout); |
| 264 | dispatchIndexOpFoldResults(mixedSgData, dynamicSgData, staticSgData); |
| 265 | dispatchIndexOpFoldResults(mixedInstData, dynamicInstData, staticInstData); |
| 266 | build(builder, ostate, target.getType(), |
| 267 | /*target=*/target, |
| 268 | /*index=*/index, |
| 269 | /*sg_layout=*/dynamicSgLayout, |
| 270 | /*sg_data=*/dynamicSgData, |
| 271 | /*inst_data=*/dynamicInstData, |
| 272 | /*packed_sg_layout=*/Value(), |
| 273 | /*packed_sg_data=*/Value(), |
| 274 | /*packed_inst_data=*/Value(), |
| 275 | /*static_sg_layout=*/staticSgLayout, |
| 276 | /*static_sg_data=*/staticSgData, |
| 277 | /*static_inst_data=*/staticInstData, |
| 278 | /*order=*/order, |
| 279 | /*slice_dims=*/sliceDims); |
| 280 | } |
| 281 | |
| 282 | LogicalResult transform::SetAnchorLayoutOp::verify() { |
| 283 | if (getPackedSgLayout() && !getMixedSgLayout().empty()) |
| 284 | return emitOpError("sg_layout and packed_sg_layout are mutually exclusive" ); |
| 285 | if (getPackedSgData() && !getMixedSgData().empty()) |
| 286 | return emitOpError("sg_data and packed_sg_data are mutually exclusive" ); |
| 287 | if (getPackedInstData() && !getMixedInstData().empty()) |
| 288 | return emitOpError("inst_data and packed_inst_data are mutually exclusive" ); |
| 289 | return success(); |
| 290 | } |
| 291 | |
| 292 | DiagnosedSilenceableFailure |
| 293 | transform::SetAnchorLayoutOp::apply(transform::TransformRewriter &rewriter, |
| 294 | transform::TransformResults &results, |
| 295 | transform::TransformState &state) { |
| 296 | auto targetOps = state.getPayloadOps(getTarget()); |
| 297 | int64_t index = getIndex(); |
| 298 | |
| 299 | // Construct layout attribute. |
| 300 | xegpu::LayoutAttr layoutAttr = nullptr; |
| 301 | auto status = getLayoutAttrFromOperands( |
| 302 | getContext(), state, (*this), getMixedSgLayout(), getMixedSgData(), |
| 303 | getMixedInstData(), getPackedSgLayout(), getPackedSgData(), |
| 304 | getPackedInstData(), getOrder(), layoutAttr); |
| 305 | if (!status.succeeded()) |
| 306 | return status; |
| 307 | |
| 308 | xegpu::DistributeLayoutAttr layout = layoutAttr; |
| 309 | auto sliceDims = getSliceDims(); |
| 310 | if (sliceDims.size() > 0) { |
| 311 | // Wrap layoutAttr in a slice attribute. |
| 312 | layout = xegpu::SliceAttr::get( |
| 313 | getContext(), layout, DenseI64ArrayAttr::get(getContext(), sliceDims)); |
| 314 | } |
| 315 | |
| 316 | // Apply the layout to all target ops. |
| 317 | for (Operation *target : targetOps) { |
| 318 | // Set layout attribute |
| 319 | if (auto dpasOp = dyn_cast<xegpu::DpasOp>(target)) { |
| 320 | // dpas op is a special case where layout needs to be set for A, B, and C |
| 321 | if (index == 0) |
| 322 | dpasOp.getProperties().layout_a = layout; |
| 323 | else if (index == 1) |
| 324 | dpasOp.getProperties().layout_b = layout; |
| 325 | else if (index == 2) |
| 326 | dpasOp.getProperties().layout_cd = layout; |
| 327 | else { |
| 328 | auto diag = emitSilenceableFailure(getLoc()) |
| 329 | << "Invalid index for setting dpas op layout: " << index; |
| 330 | diag.attachNote(target->getLoc()) << "target op" ; |
| 331 | return diag; |
| 332 | } |
| 333 | } else { |
| 334 | // op's anchor layout. |
| 335 | auto anchorOp = dyn_cast<xegpu::AnchorLayoutInterface>(target); |
| 336 | if (!anchorOp) { |
| 337 | auto diag = emitSilenceableFailure(getLoc()) |
| 338 | << "Cannot set anchor layout to op: " << target->getName(); |
| 339 | diag.attachNote(target->getLoc()) << "target op" ; |
| 340 | return diag; |
| 341 | } |
| 342 | anchorOp.setAnchorLayout(layout); |
| 343 | } |
| 344 | } |
| 345 | return DiagnosedSilenceableFailure::success(); |
| 346 | } |
| 347 | |
| 348 | void transform::SetAnchorLayoutOp::getEffects( |
| 349 | ::llvm::SmallVectorImpl<MemoryEffects::EffectInstance> &effects) { |
| 350 | onlyReadsHandle(getTargetMutable(), effects); |
| 351 | onlyReadsHandle(getSgLayoutMutable(), effects); |
| 352 | onlyReadsHandle(getSgDataMutable(), effects); |
| 353 | onlyReadsHandle(getInstDataMutable(), effects); |
| 354 | onlyReadsHandle(getPackedSgLayoutMutable(), effects); |
| 355 | onlyReadsHandle(getPackedSgDataMutable(), effects); |
| 356 | onlyReadsHandle(getPackedInstDataMutable(), effects); |
| 357 | modifiesPayload(effects); |
| 358 | } |
| 359 | |
| 360 | void transform::SetGPULaunchThreadsOp::build( |
| 361 | OpBuilder &builder, OperationState &ostate, Value target, |
| 362 | ArrayRef<OpFoldResult> mixedThreads) { |
| 363 | SmallVector<int64_t> staticThreads; |
| 364 | SmallVector<Value> dynamicThreads; |
| 365 | dispatchIndexOpFoldResults(mixedThreads, dynamicThreads, staticThreads); |
| 366 | build(builder, ostate, target.getType(), |
| 367 | /*target=*/target, |
| 368 | /*threads=*/dynamicThreads, |
| 369 | /*static_threads=*/staticThreads); |
| 370 | } |
| 371 | |
| 372 | DiagnosedSilenceableFailure |
| 373 | transform::SetGPULaunchThreadsOp::apply(transform::TransformRewriter &rewriter, |
| 374 | transform::TransformResults &results, |
| 375 | transform::TransformState &state) { |
| 376 | auto targetOps = state.getPayloadOps(getTarget()); |
| 377 | if (!llvm::hasSingleElement(targetOps)) { |
| 378 | return emitDefiniteFailure() << "Requires exactly one targetOp handle (got " |
| 379 | << llvm::range_size(targetOps) << ")" ; |
| 380 | } |
| 381 | Operation *target = *targetOps.begin(); |
| 382 | |
| 383 | auto launchOp = dyn_cast<gpu::LaunchOp>(target); |
| 384 | if (!launchOp) { |
| 385 | auto diag = emitSilenceableFailure(getLoc()) |
| 386 | << "Expected a gpu.launch op, but got: " << target->getName(); |
| 387 | diag.attachNote(target->getLoc()) << "target op" ; |
| 388 | return diag; |
| 389 | } |
| 390 | |
| 391 | SmallVector<int32_t> threads; |
| 392 | DiagnosedSilenceableFailure status = |
| 393 | convertMixedValuesToInt(state, (*this), threads, getMixedThreads()); |
| 394 | if (!status.succeeded()) |
| 395 | return status; |
| 396 | |
| 397 | if (threads.size() != 3) { |
| 398 | return emitSilenceableFailure(getLoc()) |
| 399 | << "Expected threads argument to consist of three values (got " |
| 400 | << threads.size() << ")" ; |
| 401 | } |
| 402 | |
| 403 | rewriter.setInsertionPoint(launchOp); |
| 404 | auto createConstValue = [&](int value) { |
| 405 | return arith::ConstantIndexOp::create(rewriter, launchOp.getLoc(), value); |
| 406 | }; |
| 407 | |
| 408 | // Replace threads in-place. |
| 409 | launchOp.getBlockSizeXMutable().assign(createConstValue(threads[0])); |
| 410 | launchOp.getBlockSizeYMutable().assign(createConstValue(threads[1])); |
| 411 | launchOp.getBlockSizeZMutable().assign(createConstValue(threads[2])); |
| 412 | |
| 413 | return DiagnosedSilenceableFailure::success(); |
| 414 | } |
| 415 | |
| 416 | void transform::SetGPULaunchThreadsOp::getEffects( |
| 417 | ::llvm::SmallVectorImpl<MemoryEffects::EffectInstance> &effects) { |
| 418 | onlyReadsHandle(getTargetMutable(), effects); |
| 419 | onlyReadsHandle(getThreadsMutable(), effects); |
| 420 | modifiesPayload(effects); |
| 421 | } |
| 422 | |
| 423 | DiagnosedSilenceableFailure |
| 424 | transform::InsertPrefetchOp::apply(transform::TransformRewriter &rewriter, |
| 425 | transform::TransformResults &results, |
| 426 | transform::TransformState &state) { |
| 427 | auto targetOps = state.getPayloadOps(getTarget()); |
| 428 | if (!llvm::hasSingleElement(targetOps)) |
| 429 | return emitDefiniteFailure() |
| 430 | << "requires exactly one target op handle (got " |
| 431 | << llvm::range_size(targetOps) << ")" ; |
| 432 | auto target = *targetOps.begin(); |
| 433 | |
| 434 | int64_t nbPrefetch = getStaticNbPrefetch(); |
| 435 | if (getDynamicNbPrefetch()) { |
| 436 | // Get dynamic prefetch count from transform param or handle. |
| 437 | SmallVector<int32_t> dynamicNbPrefetch; |
| 438 | auto status = convertMixedValuesToInt(state, (*this), dynamicNbPrefetch, |
| 439 | {getDynamicNbPrefetch()}); |
| 440 | if (!status.succeeded()) |
| 441 | return status; |
| 442 | if (dynamicNbPrefetch.size() != 1) |
| 443 | return emitDefiniteFailure() |
| 444 | << "requires exactly one value for dynamic_nb_prefetch" ; |
| 445 | nbPrefetch = dynamicNbPrefetch[0]; |
| 446 | } |
| 447 | if (nbPrefetch <= 0) |
| 448 | return emitSilenceableFailure(getLoc()) |
| 449 | << "nb_prefetch must be a positive integer." ; |
| 450 | |
| 451 | // Cast target to load op. |
| 452 | auto maybeLoadOp = dyn_cast<xegpu::LoadNdOp>(target); |
| 453 | if (!maybeLoadOp) { |
| 454 | return emitSilenceableFailure(getLoc()) |
| 455 | << "Expected xegpu.load_nd op, got " << target->getName(); |
| 456 | } |
| 457 | auto loadOp = maybeLoadOp; |
| 458 | if (loadOp.getMixedOffsets().size() == 0) { |
| 459 | auto diag = emitSilenceableFailure(getLoc()) |
| 460 | << "Load op must have offsets." ; |
| 461 | diag.attachNote(loadOp.getLoc()) << "load op" ; |
| 462 | return diag; |
| 463 | } |
| 464 | |
| 465 | // Find the parent scf.for loop. |
| 466 | auto forOp = loadOp->getParentOfType<scf::ForOp>(); |
| 467 | if (!forOp) { |
| 468 | auto diag = emitSilenceableFailure(getLoc()) |
| 469 | << "Load op is not contained in a scf.for loop." ; |
| 470 | diag.attachNote(loadOp.getLoc()) << "load op" ; |
| 471 | return diag; |
| 472 | } |
| 473 | |
| 474 | // Find descriptor op. |
| 475 | auto maybeDescOp = |
| 476 | findProducerOfType<xegpu::CreateNdDescOp>(loadOp.getResult()); |
| 477 | if (!maybeDescOp) |
| 478 | return emitSilenceableFailure(getLoc()) << "Could not find descriptor op." ; |
| 479 | auto descOp = *maybeDescOp; |
| 480 | |
| 481 | // Collect the loop-local operations computing the load offsets. These |
| 482 | // operations must be safe to execute before the corresponding load, both in |
| 483 | // the initial prefetch loop and at the start of each main-loop iteration. |
| 484 | SetVector<Operation *> offsetSlice; |
| 485 | BackwardSliceOptions sliceOptions; |
| 486 | sliceOptions.inclusive = true; |
| 487 | sliceOptions.omitBlockArguments = true; |
| 488 | sliceOptions.filter = [&](Operation *op) { |
| 489 | return op->getBlock() == forOp.getBody(); |
| 490 | }; |
| 491 | for (Value offset : loadOp.getOffsets()) |
| 492 | if (failed(getBackwardSlice(offset, &offsetSlice, sliceOptions))) |
| 493 | return emitSilenceableFailure(getLoc()) |
| 494 | << "Could not trace load offset computation." ; |
| 495 | |
| 496 | DominanceInfo dominance; |
| 497 | auto isAvailable = [&](Value value) { |
| 498 | return value == forOp.getInductionVar() || |
| 499 | (value.getDefiningOp() && |
| 500 | offsetSlice.contains(value.getDefiningOp())) || |
| 501 | dominance.dominates(value, forOp); |
| 502 | }; |
| 503 | for (Operation *op : offsetSlice) { |
| 504 | if (!isPure(op) || op->getNumRegions() != 0) |
| 505 | return emitSilenceableFailure(getLoc()) |
| 506 | << "Offset computation cannot be safely replicated before the " |
| 507 | "for loop." ; |
| 508 | if (!llvm::all_of(op->getOperands(), isAvailable)) |
| 509 | return emitSilenceableFailure(getLoc()) |
| 510 | << "Load offset depends on a value unavailable before the loop." ; |
| 511 | } |
| 512 | if (!llvm::all_of(loadOp.getOffsets(), isAvailable)) |
| 513 | return emitSilenceableFailure(getLoc()) |
| 514 | << "Load offset depends on a value unavailable before the loop." ; |
| 515 | |
| 516 | // Clone desc op outside the loop. |
| 517 | rewriter.setInsertionPoint(forOp); |
| 518 | auto newDescOp = |
| 519 | cast<xegpu::CreateNdDescOp>(rewriter.clone(*descOp.getOperation())); |
| 520 | |
| 521 | // Clone reduction loop to emit initial prefetches. |
| 522 | // Compute upper bound of the init loop: start + nbPrefetch * step. |
| 523 | auto nbPrefetchCst = |
| 524 | arith::ConstantIndexOp::create(rewriter, forOp.getLoc(), nbPrefetch); |
| 525 | auto nbStep = rewriter.createOrFold<arith::MulIOp>( |
| 526 | forOp.getLoc(), nbPrefetchCst, forOp.getStep()); |
| 527 | auto initUpBound = rewriter.createOrFold<arith::AddIOp>( |
| 528 | forOp.getLoc(), forOp.getLowerBound(), nbStep); |
| 529 | auto initForOp = |
| 530 | scf::ForOp::create(rewriter, forOp.getLoc(), forOp.getLowerBound(), |
| 531 | initUpBound, forOp.getStep()); |
| 532 | |
| 533 | auto ctx = rewriter.getContext(); |
| 534 | auto readCacheHint = |
| 535 | xegpu::CachePolicyAttr::get(ctx, xegpu::CachePolicy::CACHED); |
| 536 | |
| 537 | // Recreate the offset calculation with the induction variable replaced by |
| 538 | // the iteration being prefetched. Cloning in slice order maps every result |
| 539 | // before it is used by a later offset operation. |
| 540 | auto getPrefetchOffsets = |
| 541 | [&](Value replacementVal) -> SmallVector<OpFoldResult> { |
| 542 | IRMapping mapping; |
| 543 | mapping.map(forOp.getInductionVar(), replacementVal); |
| 544 | for (Operation *op : offsetSlice) |
| 545 | rewriter.clone(*op, mapping); |
| 546 | SmallVector<Value> dynamicOffsets = |
| 547 | llvm::map_to_vector(loadOp.getOffsets(), [&](Value v) { |
| 548 | return mapping.lookupOrDefault(v); |
| 549 | }); |
| 550 | auto constOffsets = loadOp.getConstOffsets(); |
| 551 | return getMixedValues(constOffsets, dynamicOffsets, ctx); |
| 552 | }; |
| 553 | |
| 554 | // Insert prefetch op in init loop. |
| 555 | // Replace induction var with the init loop induction var. |
| 556 | rewriter.setInsertionPointToStart(initForOp.getBody()); |
| 557 | xegpu::PrefetchNdOp::create(rewriter, newDescOp.getLoc(), |
| 558 | newDescOp.getResult(), |
| 559 | getPrefetchOffsets(initForOp.getInductionVar()), |
| 560 | readCacheHint, readCacheHint, readCacheHint, |
| 561 | /*layout=*/nullptr); |
| 562 | |
| 563 | // Insert prefetch op in main loop. |
| 564 | // Calculate prefetch offset after the init prefetches have been issued. |
| 565 | rewriter.setInsertionPointToStart(forOp.getBody()); |
| 566 | auto prefetchOffset = arith::AddIOp::create(rewriter, forOp.getLoc(), |
| 567 | forOp.getInductionVar(), nbStep); |
| 568 | // Replace induction var with correct offset. |
| 569 | xegpu::PrefetchNdOp::create(rewriter, newDescOp.getLoc(), |
| 570 | newDescOp.getResult(), |
| 571 | getPrefetchOffsets(prefetchOffset), readCacheHint, |
| 572 | readCacheHint, readCacheHint, /*layout=*/nullptr); |
| 573 | |
| 574 | // Unroll the init loop. |
| 575 | if (failed(loopUnrollFull(initForOp))) |
| 576 | return emitSilenceableFailure(getLoc()) << "Failed to unroll the loop" ; |
| 577 | |
| 578 | results.set(llvm::cast<OpResult>(getResult()), {newDescOp}); |
| 579 | |
| 580 | return DiagnosedSilenceableFailure::success(); |
| 581 | } |
| 582 | |
| 583 | void transform::InsertPrefetchOp::getEffects( |
| 584 | ::llvm::SmallVectorImpl<MemoryEffects::EffectInstance> &effects) { |
| 585 | onlyReadsHandle(getTargetMutable(), effects); |
| 586 | onlyReadsHandle(getDynamicNbPrefetchMutable(), effects); |
| 587 | producesHandle(getOperation()->getOpResults(), effects); |
| 588 | modifiesPayload(effects); |
| 589 | } |
| 590 | |
| 591 | void transform::ConvertLayoutOp::build( |
| 592 | OpBuilder &builder, OperationState &ostate, Value target, |
| 593 | ArrayRef<OpFoldResult> mixedInputSgLayout, |
| 594 | ArrayRef<OpFoldResult> mixedInputSgData, |
| 595 | ArrayRef<OpFoldResult> mixedInputInstData, ArrayRef<int32_t> inputOrder, |
| 596 | ArrayRef<OpFoldResult> mixedTargetSgLayout, |
| 597 | ArrayRef<OpFoldResult> mixedTargetSgData, |
| 598 | ArrayRef<OpFoldResult> mixedTargetInstData, ArrayRef<int32_t> targetOrder) { |
| 599 | SmallVector<int64_t> staticInputSgLayout, staticInputSgData, |
| 600 | staticInputInstData; |
| 601 | SmallVector<Value> dynamicInputSgLayout, dynamicInputSgData, |
| 602 | dynamicInputInstData; |
| 603 | dispatchIndexOpFoldResults(ofrs: mixedInputSgLayout, dynamicVec&: dynamicInputSgLayout, |
| 604 | staticVec&: staticInputSgLayout); |
| 605 | dispatchIndexOpFoldResults(ofrs: mixedInputSgData, dynamicVec&: dynamicInputSgData, |
| 606 | staticVec&: staticInputSgData); |
| 607 | dispatchIndexOpFoldResults(ofrs: mixedInputInstData, dynamicVec&: dynamicInputInstData, |
| 608 | staticVec&: staticInputInstData); |
| 609 | SmallVector<int64_t> staticTargetSgLayout, staticTargetSgData, |
| 610 | staticTargetInstData; |
| 611 | SmallVector<Value> dynamicTargetSgLayout, dynamicTargetSgData, |
| 612 | dynamicTargetInstData; |
| 613 | dispatchIndexOpFoldResults(ofrs: mixedTargetSgLayout, dynamicVec&: dynamicTargetSgLayout, |
| 614 | staticVec&: staticTargetSgLayout); |
| 615 | dispatchIndexOpFoldResults(ofrs: mixedTargetSgData, dynamicVec&: dynamicTargetSgData, |
| 616 | staticVec&: staticTargetSgData); |
| 617 | dispatchIndexOpFoldResults(ofrs: mixedTargetInstData, dynamicVec&: dynamicTargetInstData, |
| 618 | staticVec&: staticTargetInstData); |
| 619 | build(builder, ostate, target.getType(), |
| 620 | /*target=*/target, |
| 621 | /*input_sg_layout=*/dynamicInputSgLayout, |
| 622 | /*input_sg_data=*/dynamicInputSgData, |
| 623 | /*input_inst_data=*/dynamicInputInstData, |
| 624 | /*target_sg_layout=*/dynamicTargetSgLayout, |
| 625 | /*target_sg_data=*/dynamicTargetSgData, |
| 626 | /*target_inst_data=*/dynamicTargetInstData, |
| 627 | /*input_order=*/inputOrder, |
| 628 | /*static_input_sg_layout=*/staticInputSgLayout, |
| 629 | /*static_input_sg_data=*/staticInputSgData, |
| 630 | /*static_input_inst_data=*/staticInputInstData, |
| 631 | /*static_target_sg_layout=*/staticTargetSgLayout, |
| 632 | /*static_target_sg_data=*/staticTargetSgData, |
| 633 | /*static_target_inst_data=*/staticTargetInstData, |
| 634 | /*target_order=*/targetOrder); |
| 635 | } |
| 636 | |
| 637 | DiagnosedSilenceableFailure |
| 638 | transform::ConvertLayoutOp::apply(transform::TransformRewriter &rewriter, |
| 639 | transform::TransformResults &results, |
| 640 | transform::TransformState &state) { |
| 641 | auto targetValues = state.getPayloadValues(handleValue: getTarget()); |
| 642 | if (!llvm::hasSingleElement(targetValues)) |
| 643 | return emitDefiniteFailure() |
| 644 | << "requires exactly one target value handle (got " |
| 645 | << llvm::range_size(targetValues) << ")" ; |
| 646 | auto value = *targetValues.begin(); |
| 647 | |
| 648 | // Construct layout attributes. |
| 649 | xegpu::LayoutAttr inputLayoutAttr = nullptr; |
| 650 | auto status = getLayoutAttrFromOperands( |
| 651 | getContext(), state, (*this), getMixedInputSgLayout(), |
| 652 | getMixedInputSgData(), getMixedInputInstData(), |
| 653 | /*packedSgLayout=*/Value(), |
| 654 | /*packedSgData=*/Value(), /*packedInstData=*/Value(), getInputOrder(), |
| 655 | inputLayoutAttr); |
| 656 | if (!status.succeeded()) |
| 657 | return status; |
| 658 | |
| 659 | xegpu::LayoutAttr targetLayoutAttr = nullptr; |
| 660 | status = getLayoutAttrFromOperands( |
| 661 | getContext(), state, (*this), getMixedTargetSgLayout(), |
| 662 | getMixedTargetSgData(), getMixedTargetInstData(), |
| 663 | /*packedSgLayout=*/Value(), |
| 664 | /*packedSgData=*/Value(), /*packedInstData=*/Value(), getTargetOrder(), |
| 665 | targetLayoutAttr); |
| 666 | if (!status.succeeded()) |
| 667 | return status; |
| 668 | |
| 669 | // Find first user op to define insertion point for layout conversion. |
| 670 | if (value.use_empty()) |
| 671 | return emitSilenceableFailure(loc: getLoc()) |
| 672 | << "Value has no users to insert layout conversion." ; |
| 673 | Operation *userOp = *value.getUsers().begin(); |
| 674 | |
| 675 | // Emit convert_layout op. |
| 676 | rewriter.setInsertionPoint(userOp); |
| 677 | auto convLayoutOp = |
| 678 | xegpu::ConvertLayoutOp::create(rewriter, value.getLoc(), value.getType(), |
| 679 | value, inputLayoutAttr, targetLayoutAttr); |
| 680 | // Replace load op result with the converted layout. |
| 681 | rewriter.replaceUsesWithIf( |
| 682 | value, convLayoutOp.getResult(), [&](OpOperand &use) { |
| 683 | return use.getOwner() != convLayoutOp.getOperation(); |
| 684 | }); |
| 685 | |
| 686 | results.set(llvm::cast<OpResult>(Val: getResult()), {convLayoutOp}); |
| 687 | return DiagnosedSilenceableFailure::success(); |
| 688 | } |
| 689 | |
| 690 | void transform::ConvertLayoutOp::getEffects( |
| 691 | ::llvm::SmallVectorImpl<MemoryEffects::EffectInstance> &effects) { |
| 692 | onlyReadsHandle(handles: getTargetMutable(), effects); |
| 693 | onlyReadsHandle(getInputSgLayoutMutable(), effects); |
| 694 | onlyReadsHandle(getInputSgDataMutable(), effects); |
| 695 | onlyReadsHandle(getInputInstDataMutable(), effects); |
| 696 | onlyReadsHandle(getTargetSgLayoutMutable(), effects); |
| 697 | onlyReadsHandle(getTargetSgDataMutable(), effects); |
| 698 | onlyReadsHandle(getTargetInstDataMutable(), effects); |
| 699 | producesHandle(handles: getOperation()->getOpResults(), effects); |
| 700 | modifiesPayload(effects); |
| 701 | } |
| 702 | |
| 703 | namespace { |
| 704 | class XeGPUTransformDialectExtension |
| 705 | : public transform::TransformDialectExtension< |
| 706 | XeGPUTransformDialectExtension> { |
| 707 | public: |
| 708 | MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(XeGPUTransformDialectExtension) |
| 709 | |
| 710 | using Base::Base; |
| 711 | |
| 712 | void init(); |
| 713 | }; |
| 714 | |
| 715 | void XeGPUTransformDialectExtension::init() { |
| 716 | declareGeneratedDialect<scf::SCFDialect>(); |
| 717 | declareGeneratedDialect<arith::ArithDialect>(); |
| 718 | declareGeneratedDialect<xegpu::XeGPUDialect>(); |
| 719 | |
| 720 | registerTransformOps< |
| 721 | #define GET_OP_LIST |
| 722 | #include "mlir/Dialect/XeGPU/TransformOps/XeGPUTransformOps.cpp.inc" |
| 723 | >(); |
| 724 | } |
| 725 | } // namespace |
| 726 | |
| 727 | #define GET_OP_CLASSES |
| 728 | #include "mlir/Dialect/XeGPU/TransformOps/XeGPUTransformOps.cpp.inc" |
| 729 | |
| 730 | void mlir::xegpu::registerTransformDialectExtension(DialectRegistry ®istry) { |
| 731 | registry.addExtensions<XeGPUTransformDialectExtension>(); |
| 732 | } |
| 733 | |