1//===- XeGPUTransformOps.cpp - Implementation of XeGPU transformation ops -===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Dialect/XeGPU/TransformOps/XeGPUTransformOps.h"
10#include "mlir/Analysis/SliceAnalysis.h"
11#include "mlir/Dialect/GPU/IR/GPUDialect.h"
12#include "mlir/Dialect/SCF/IR/SCF.h"
13#include "mlir/Dialect/SCF/Utils/Utils.h"
14#include "mlir/Dialect/Transform/Utils/Utils.h"
15#include "mlir/Dialect/XeGPU/IR/XeGPU.h"
16#include "mlir/Dialect/XeGPU/Utils/XeGPUUtils.h"
17#include "mlir/IR/Dominance.h"
18#include "mlir/Interfaces/SideEffectInterfaces.h"
19#include "llvm/ADT/SmallVectorExtras.h"
20
21#include <optional>
22
23#include "llvm/Support/DebugLog.h"
24#define DEBUG_TYPE "xegpu-transforms"
25
26using namespace mlir;
27using namespace mlir::transform;
28
29/// Assuming that `ofr` is an index attr or a param of index type
30/// or a transform dialect handle mapped to exactly one op
31/// with one index result, get that value and cast it to int type.
32static DiagnosedSilenceableFailure convertMixedValuesToInt(
33 transform::TransformState &state, TransformOpInterface transformOp,
34 SmallVectorImpl<int32_t> &result, ArrayRef<OpFoldResult> ofrs) {
35 for (OpFoldResult ofr : ofrs) {
36 // Attribute case.
37 if (auto attr = dyn_cast<Attribute>(Val&: ofr)) {
38 if (auto intAttr = dyn_cast<IntegerAttr>(Val&: attr)) {
39 result.push_back(Elt: intAttr.getInt());
40 continue;
41 }
42 return transformOp.emitDefiniteFailure() << "expected IntegerAttr";
43 }
44
45 // Transform param case.
46 Value transformValue = cast<Value>(Val&: ofr);
47 if (isa<TransformParamTypeInterface>(Val: transformValue.getType())) {
48 ArrayRef<Attribute> params = state.getParams(value: transformValue);
49 if (params.size() != 1)
50 return transformOp.emitDefiniteFailure()
51 << "requires exactly one parameter associated";
52 result.push_back(
53 Elt: cast<IntegerAttr>(Val: params.front()).getValue().getSExtValue());
54 continue;
55 }
56
57 // Payload value case.
58 auto payloadOps = state.getPayloadOps(value: transformValue);
59 if (!llvm::hasSingleElement(C&: payloadOps)) {
60 DiagnosedSilenceableFailure diag =
61 transformOp.emitSilenceableError()
62 << "handle must be mapped to exactly one payload op";
63 diag.attachNote(loc: transformValue.getLoc())
64 << "mapped to " << llvm::range_size(Range&: payloadOps) << " payload ops";
65 return diag;
66 }
67
68 Operation *op = *payloadOps.begin();
69 if (op->getNumResults() != 1 || !op->getResult(idx: 0).getType().isIndex()) {
70 DiagnosedSilenceableFailure diag =
71 transformOp.emitSilenceableError()
72 << "payload op must have exactly 1 index result";
73 diag.attachNote(loc: op->getLoc())
74 << "has " << op->getNumResults() << " results";
75 return diag;
76 }
77
78 IntegerAttr intAttr;
79 if (!matchPattern(value: op->getResult(idx: 0), pattern: m_Constant(bind_value: &intAttr)))
80 return transformOp.emitSilenceableError()
81 << "requires param or handle to be the result of a constant like "
82 "op";
83
84 result.push_back(Elt: intAttr.getInt());
85 }
86 return DiagnosedSilenceableFailure::success();
87}
88
89/// Convert a single packed handle, that is either a transform param associated
90/// with a variadic number of integer attributes or mapped to a variadic number
91/// of payload ops each defining a single constant index result, to a list of
92/// int values.
93static DiagnosedSilenceableFailure
94convertPackedValueToInt(transform::TransformState &state,
95 TransformOpInterface transformOp,
96 SmallVectorImpl<int32_t> &result, Value packedHandle) {
97 // Transform param case.
98 if (isa<TransformParamTypeInterface>(Val: packedHandle.getType())) {
99 for (Attribute param : state.getParams(value: packedHandle)) {
100 auto intAttr = dyn_cast<IntegerAttr>(Val&: param);
101 if (!intAttr)
102 return transformOp.emitDefiniteFailure()
103 << "expected the parameter to be associated with an integer "
104 "attribute";
105 result.push_back(Elt: intAttr.getValue().getSExtValue());
106 }
107 return DiagnosedSilenceableFailure::success();
108 }
109
110 // Payload op case.
111 for (Operation *op : state.getPayloadOps(value: packedHandle)) {
112 if (op->getNumResults() != 1 || !op->getResult(idx: 0).getType().isIndex()) {
113 DiagnosedSilenceableFailure diag =
114 transformOp.emitSilenceableError()
115 << "payload op must have exactly 1 index result";
116 diag.attachNote(loc: op->getLoc())
117 << "has " << op->getNumResults() << " results";
118 return diag;
119 }
120
121 IntegerAttr intAttr;
122 if (!matchPattern(value: op->getResult(idx: 0), pattern: m_Constant(bind_value: &intAttr)))
123 return transformOp.emitSilenceableError()
124 << "requires param or handle to be the result of a constant like "
125 "op";
126
127 result.push_back(Elt: intAttr.getInt());
128 }
129 return DiagnosedSilenceableFailure::success();
130}
131
132/// It's assumed that producer ops are chained through their first operand.
133/// Producer chain is traced trough loop block arguments (init values).
134template <typename T>
135static std::optional<T> findProducerOfType(Value val) {
136 Value currentValue = val;
137 if (!currentValue.getDefiningOp()) {
138 // Value may be a block argument initialized outside a loop.
139 if (val.getNumUses() == 0) {
140 LDBG() << "Failed to find producer op, value has no uses.";
141 return std::nullopt;
142 }
143 auto userOp = val.getUsers().begin();
144 auto parentLoop = userOp->getParentOfType<LoopLikeOpInterface>();
145 if (!parentLoop) {
146 LDBG() << "Failed to find producer op, not in a loop.";
147 return std::nullopt;
148 }
149 int64_t iterArgIdx;
150 if (auto iterArg = llvm::dyn_cast<BlockArgument>(Val&: currentValue)) {
151 auto numInductionVars = parentLoop.getLoopInductionVars()->size();
152 iterArgIdx = iterArg.getArgNumber() - numInductionVars;
153 currentValue = parentLoop.getInits()[iterArgIdx];
154 } else {
155 LDBG() << "Failed to find producer op, value not in init values.";
156 return std::nullopt;
157 }
158 }
159 Operation *producerOp = currentValue.getDefiningOp();
160
161 if (auto matchingOp = dyn_cast<T>(producerOp))
162 return matchingOp;
163
164 if (producerOp->getNumOperands() == 0)
165 return std::nullopt;
166
167 return findProducerOfType<T>(producerOp->getOperand(idx: 0));
168}
169
170/// Create a layout attribute from the given parameters.
171static xegpu::LayoutAttr createLayoutAttr(
172 MLIRContext *ctx, ArrayRef<int32_t> sgLayout, ArrayRef<int32_t> sgData,
173 std::optional<ArrayRef<int32_t>> instData, ArrayRef<int32_t> order) {
174 return xegpu::LayoutAttr::get(
175 context: ctx, sg_layout: DenseI32ArrayAttr::get(context: ctx, content: sgLayout),
176 sg_data: DenseI32ArrayAttr::get(context: ctx, content: sgData),
177 inst_data: instData ? DenseI32ArrayAttr::get(context: ctx, content: instData.value()) : nullptr,
178 /*lane_layout=*/nullptr,
179 /*lane_data=*/nullptr,
180 /*order=*/order.empty() ? nullptr : DenseI32ArrayAttr::get(context: ctx, content: order));
181}
182
183/// Generate `xegpu::LayoutAttr` from op mixed layout values.
184DiagnosedSilenceableFailure
185getLayoutAttrFromOperands(MLIRContext *ctx, transform::TransformState &state,
186 TransformOpInterface transformOp,
187 ArrayRef<::mlir::OpFoldResult> mixedSgLayout,
188 ArrayRef<::mlir::OpFoldResult> mixedSgData,
189 ArrayRef<::mlir::OpFoldResult> mixedInstData,
190 Value packedSgLayout, Value packedSgData,
191 Value packedInstData, ArrayRef<int32_t> order,
192 xegpu::LayoutAttr &layoutAttr) {
193 SmallVector<int32_t> sgLayout, sgData, instData;
194
195 // A packed handle, when present, supersedes the mixed values and is unpacked
196 // to a variadic list of integers.
197 auto convert = [&](Value packed, ArrayRef<OpFoldResult> mixed,
198 SmallVectorImpl<int32_t> &out) {
199 return packed ? convertPackedValueToInt(state, transformOp, result&: out, packedHandle: packed)
200 : convertMixedValuesToInt(state, transformOp, result&: out, ofrs: mixed);
201 };
202
203 auto status = convert(packedSgLayout, mixedSgLayout, sgLayout);
204 if (!status.succeeded())
205 return status;
206
207 status = convert(packedSgData, mixedSgData, sgData);
208 if (!status.succeeded())
209 return status;
210
211 status = convert(packedInstData, mixedInstData, instData);
212 if (!status.succeeded())
213 return status;
214 auto maybeInstData = instData.empty()
215 ? std::nullopt
216 : std::optional<ArrayRef<int32_t>>(instData);
217
218 layoutAttr = createLayoutAttr(ctx, sgLayout, sgData, instData: maybeInstData, order);
219
220 return DiagnosedSilenceableFailure::success();
221}
222
223DiagnosedSilenceableFailure
224transform::GetLoadOp::apply(transform::TransformRewriter &rewriter,
225 transform::TransformResults &results,
226 transform::TransformState &state) {
227 auto targetValues = state.getPayloadValues(getTarget());
228 if (!llvm::hasSingleElement(targetValues)) {
229 return emitDefiniteFailure()
230 << "requires exactly one target value handle (got "
231 << llvm::range_size(targetValues) << ")";
232 }
233
234 Operation *loadOp = nullptr;
235 auto maybeLoadNdOp =
236 findProducerOfType<xegpu::LoadNdOp>(*targetValues.begin());
237 if (maybeLoadNdOp) {
238 loadOp = maybeLoadNdOp->getOperation();
239 } else {
240 auto maybeLoadOp =
241 findProducerOfType<xegpu::LoadGatherOp>(*targetValues.begin());
242 if (maybeLoadOp) {
243 loadOp = maybeLoadOp->getOperation();
244 } else {
245 return emitSilenceableFailure(getLoc())
246 << "Could not find a matching xegpu.load_nd or xegpu.load op when "
247 "walking the "
248 "producer chain of the first operand.";
249 }
250 }
251
252 results.set(llvm::cast<OpResult>(getResult()), {loadOp});
253 return DiagnosedSilenceableFailure::success();
254}
255
256void transform::SetAnchorLayoutOp::build(
257 OpBuilder &builder, OperationState &ostate, Value target, int64_t index,
258 ArrayRef<OpFoldResult> mixedSgLayout, ArrayRef<OpFoldResult> mixedSgData,
259 ArrayRef<OpFoldResult> mixedInstData, ArrayRef<int32_t> order,
260 ArrayRef<int64_t> sliceDims) {
261 SmallVector<int64_t> staticSgLayout, staticSgData, staticInstData;
262 SmallVector<Value> dynamicSgLayout, dynamicSgData, dynamicInstData;
263 dispatchIndexOpFoldResults(mixedSgLayout, dynamicSgLayout, staticSgLayout);
264 dispatchIndexOpFoldResults(mixedSgData, dynamicSgData, staticSgData);
265 dispatchIndexOpFoldResults(mixedInstData, dynamicInstData, staticInstData);
266 build(builder, ostate, target.getType(),
267 /*target=*/target,
268 /*index=*/index,
269 /*sg_layout=*/dynamicSgLayout,
270 /*sg_data=*/dynamicSgData,
271 /*inst_data=*/dynamicInstData,
272 /*packed_sg_layout=*/Value(),
273 /*packed_sg_data=*/Value(),
274 /*packed_inst_data=*/Value(),
275 /*static_sg_layout=*/staticSgLayout,
276 /*static_sg_data=*/staticSgData,
277 /*static_inst_data=*/staticInstData,
278 /*order=*/order,
279 /*slice_dims=*/sliceDims);
280}
281
282LogicalResult transform::SetAnchorLayoutOp::verify() {
283 if (getPackedSgLayout() && !getMixedSgLayout().empty())
284 return emitOpError("sg_layout and packed_sg_layout are mutually exclusive");
285 if (getPackedSgData() && !getMixedSgData().empty())
286 return emitOpError("sg_data and packed_sg_data are mutually exclusive");
287 if (getPackedInstData() && !getMixedInstData().empty())
288 return emitOpError("inst_data and packed_inst_data are mutually exclusive");
289 return success();
290}
291
292DiagnosedSilenceableFailure
293transform::SetAnchorLayoutOp::apply(transform::TransformRewriter &rewriter,
294 transform::TransformResults &results,
295 transform::TransformState &state) {
296 auto targetOps = state.getPayloadOps(getTarget());
297 int64_t index = getIndex();
298
299 // Construct layout attribute.
300 xegpu::LayoutAttr layoutAttr = nullptr;
301 auto status = getLayoutAttrFromOperands(
302 getContext(), state, (*this), getMixedSgLayout(), getMixedSgData(),
303 getMixedInstData(), getPackedSgLayout(), getPackedSgData(),
304 getPackedInstData(), getOrder(), layoutAttr);
305 if (!status.succeeded())
306 return status;
307
308 xegpu::DistributeLayoutAttr layout = layoutAttr;
309 auto sliceDims = getSliceDims();
310 if (sliceDims.size() > 0) {
311 // Wrap layoutAttr in a slice attribute.
312 layout = xegpu::SliceAttr::get(
313 getContext(), layout, DenseI64ArrayAttr::get(getContext(), sliceDims));
314 }
315
316 // Apply the layout to all target ops.
317 for (Operation *target : targetOps) {
318 // Set layout attribute
319 if (auto dpasOp = dyn_cast<xegpu::DpasOp>(target)) {
320 // dpas op is a special case where layout needs to be set for A, B, and C
321 if (index == 0)
322 dpasOp.getProperties().layout_a = layout;
323 else if (index == 1)
324 dpasOp.getProperties().layout_b = layout;
325 else if (index == 2)
326 dpasOp.getProperties().layout_cd = layout;
327 else {
328 auto diag = emitSilenceableFailure(getLoc())
329 << "Invalid index for setting dpas op layout: " << index;
330 diag.attachNote(target->getLoc()) << "target op";
331 return diag;
332 }
333 } else {
334 // op's anchor layout.
335 auto anchorOp = dyn_cast<xegpu::AnchorLayoutInterface>(target);
336 if (!anchorOp) {
337 auto diag = emitSilenceableFailure(getLoc())
338 << "Cannot set anchor layout to op: " << target->getName();
339 diag.attachNote(target->getLoc()) << "target op";
340 return diag;
341 }
342 anchorOp.setAnchorLayout(layout);
343 }
344 }
345 return DiagnosedSilenceableFailure::success();
346}
347
348void transform::SetAnchorLayoutOp::getEffects(
349 ::llvm::SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
350 onlyReadsHandle(getTargetMutable(), effects);
351 onlyReadsHandle(getSgLayoutMutable(), effects);
352 onlyReadsHandle(getSgDataMutable(), effects);
353 onlyReadsHandle(getInstDataMutable(), effects);
354 onlyReadsHandle(getPackedSgLayoutMutable(), effects);
355 onlyReadsHandle(getPackedSgDataMutable(), effects);
356 onlyReadsHandle(getPackedInstDataMutable(), effects);
357 modifiesPayload(effects);
358}
359
360void transform::SetGPULaunchThreadsOp::build(
361 OpBuilder &builder, OperationState &ostate, Value target,
362 ArrayRef<OpFoldResult> mixedThreads) {
363 SmallVector<int64_t> staticThreads;
364 SmallVector<Value> dynamicThreads;
365 dispatchIndexOpFoldResults(mixedThreads, dynamicThreads, staticThreads);
366 build(builder, ostate, target.getType(),
367 /*target=*/target,
368 /*threads=*/dynamicThreads,
369 /*static_threads=*/staticThreads);
370}
371
372DiagnosedSilenceableFailure
373transform::SetGPULaunchThreadsOp::apply(transform::TransformRewriter &rewriter,
374 transform::TransformResults &results,
375 transform::TransformState &state) {
376 auto targetOps = state.getPayloadOps(getTarget());
377 if (!llvm::hasSingleElement(targetOps)) {
378 return emitDefiniteFailure() << "Requires exactly one targetOp handle (got "
379 << llvm::range_size(targetOps) << ")";
380 }
381 Operation *target = *targetOps.begin();
382
383 auto launchOp = dyn_cast<gpu::LaunchOp>(target);
384 if (!launchOp) {
385 auto diag = emitSilenceableFailure(getLoc())
386 << "Expected a gpu.launch op, but got: " << target->getName();
387 diag.attachNote(target->getLoc()) << "target op";
388 return diag;
389 }
390
391 SmallVector<int32_t> threads;
392 DiagnosedSilenceableFailure status =
393 convertMixedValuesToInt(state, (*this), threads, getMixedThreads());
394 if (!status.succeeded())
395 return status;
396
397 if (threads.size() != 3) {
398 return emitSilenceableFailure(getLoc())
399 << "Expected threads argument to consist of three values (got "
400 << threads.size() << ")";
401 }
402
403 rewriter.setInsertionPoint(launchOp);
404 auto createConstValue = [&](int value) {
405 return arith::ConstantIndexOp::create(rewriter, launchOp.getLoc(), value);
406 };
407
408 // Replace threads in-place.
409 launchOp.getBlockSizeXMutable().assign(createConstValue(threads[0]));
410 launchOp.getBlockSizeYMutable().assign(createConstValue(threads[1]));
411 launchOp.getBlockSizeZMutable().assign(createConstValue(threads[2]));
412
413 return DiagnosedSilenceableFailure::success();
414}
415
416void transform::SetGPULaunchThreadsOp::getEffects(
417 ::llvm::SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
418 onlyReadsHandle(getTargetMutable(), effects);
419 onlyReadsHandle(getThreadsMutable(), effects);
420 modifiesPayload(effects);
421}
422
423DiagnosedSilenceableFailure
424transform::InsertPrefetchOp::apply(transform::TransformRewriter &rewriter,
425 transform::TransformResults &results,
426 transform::TransformState &state) {
427 auto targetOps = state.getPayloadOps(getTarget());
428 if (!llvm::hasSingleElement(targetOps))
429 return emitDefiniteFailure()
430 << "requires exactly one target op handle (got "
431 << llvm::range_size(targetOps) << ")";
432 auto target = *targetOps.begin();
433
434 int64_t nbPrefetch = getStaticNbPrefetch();
435 if (getDynamicNbPrefetch()) {
436 // Get dynamic prefetch count from transform param or handle.
437 SmallVector<int32_t> dynamicNbPrefetch;
438 auto status = convertMixedValuesToInt(state, (*this), dynamicNbPrefetch,
439 {getDynamicNbPrefetch()});
440 if (!status.succeeded())
441 return status;
442 if (dynamicNbPrefetch.size() != 1)
443 return emitDefiniteFailure()
444 << "requires exactly one value for dynamic_nb_prefetch";
445 nbPrefetch = dynamicNbPrefetch[0];
446 }
447 if (nbPrefetch <= 0)
448 return emitSilenceableFailure(getLoc())
449 << "nb_prefetch must be a positive integer.";
450
451 // Cast target to load op.
452 auto maybeLoadOp = dyn_cast<xegpu::LoadNdOp>(target);
453 if (!maybeLoadOp) {
454 return emitSilenceableFailure(getLoc())
455 << "Expected xegpu.load_nd op, got " << target->getName();
456 }
457 auto loadOp = maybeLoadOp;
458 if (loadOp.getMixedOffsets().size() == 0) {
459 auto diag = emitSilenceableFailure(getLoc())
460 << "Load op must have offsets.";
461 diag.attachNote(loadOp.getLoc()) << "load op";
462 return diag;
463 }
464
465 // Find the parent scf.for loop.
466 auto forOp = loadOp->getParentOfType<scf::ForOp>();
467 if (!forOp) {
468 auto diag = emitSilenceableFailure(getLoc())
469 << "Load op is not contained in a scf.for loop.";
470 diag.attachNote(loadOp.getLoc()) << "load op";
471 return diag;
472 }
473
474 // Find descriptor op.
475 auto maybeDescOp =
476 findProducerOfType<xegpu::CreateNdDescOp>(loadOp.getResult());
477 if (!maybeDescOp)
478 return emitSilenceableFailure(getLoc()) << "Could not find descriptor op.";
479 auto descOp = *maybeDescOp;
480
481 // Collect the loop-local operations computing the load offsets. These
482 // operations must be safe to execute before the corresponding load, both in
483 // the initial prefetch loop and at the start of each main-loop iteration.
484 SetVector<Operation *> offsetSlice;
485 BackwardSliceOptions sliceOptions;
486 sliceOptions.inclusive = true;
487 sliceOptions.omitBlockArguments = true;
488 sliceOptions.filter = [&](Operation *op) {
489 return op->getBlock() == forOp.getBody();
490 };
491 for (Value offset : loadOp.getOffsets())
492 if (failed(getBackwardSlice(offset, &offsetSlice, sliceOptions)))
493 return emitSilenceableFailure(getLoc())
494 << "Could not trace load offset computation.";
495
496 DominanceInfo dominance;
497 auto isAvailable = [&](Value value) {
498 return value == forOp.getInductionVar() ||
499 (value.getDefiningOp() &&
500 offsetSlice.contains(value.getDefiningOp())) ||
501 dominance.dominates(value, forOp);
502 };
503 for (Operation *op : offsetSlice) {
504 if (!isPure(op) || op->getNumRegions() != 0)
505 return emitSilenceableFailure(getLoc())
506 << "Offset computation cannot be safely replicated before the "
507 "for loop.";
508 if (!llvm::all_of(op->getOperands(), isAvailable))
509 return emitSilenceableFailure(getLoc())
510 << "Load offset depends on a value unavailable before the loop.";
511 }
512 if (!llvm::all_of(loadOp.getOffsets(), isAvailable))
513 return emitSilenceableFailure(getLoc())
514 << "Load offset depends on a value unavailable before the loop.";
515
516 // Clone desc op outside the loop.
517 rewriter.setInsertionPoint(forOp);
518 auto newDescOp =
519 cast<xegpu::CreateNdDescOp>(rewriter.clone(*descOp.getOperation()));
520
521 // Clone reduction loop to emit initial prefetches.
522 // Compute upper bound of the init loop: start + nbPrefetch * step.
523 auto nbPrefetchCst =
524 arith::ConstantIndexOp::create(rewriter, forOp.getLoc(), nbPrefetch);
525 auto nbStep = rewriter.createOrFold<arith::MulIOp>(
526 forOp.getLoc(), nbPrefetchCst, forOp.getStep());
527 auto initUpBound = rewriter.createOrFold<arith::AddIOp>(
528 forOp.getLoc(), forOp.getLowerBound(), nbStep);
529 auto initForOp =
530 scf::ForOp::create(rewriter, forOp.getLoc(), forOp.getLowerBound(),
531 initUpBound, forOp.getStep());
532
533 auto ctx = rewriter.getContext();
534 auto readCacheHint =
535 xegpu::CachePolicyAttr::get(ctx, xegpu::CachePolicy::CACHED);
536
537 // Recreate the offset calculation with the induction variable replaced by
538 // the iteration being prefetched. Cloning in slice order maps every result
539 // before it is used by a later offset operation.
540 auto getPrefetchOffsets =
541 [&](Value replacementVal) -> SmallVector<OpFoldResult> {
542 IRMapping mapping;
543 mapping.map(forOp.getInductionVar(), replacementVal);
544 for (Operation *op : offsetSlice)
545 rewriter.clone(*op, mapping);
546 SmallVector<Value> dynamicOffsets =
547 llvm::map_to_vector(loadOp.getOffsets(), [&](Value v) {
548 return mapping.lookupOrDefault(v);
549 });
550 auto constOffsets = loadOp.getConstOffsets();
551 return getMixedValues(constOffsets, dynamicOffsets, ctx);
552 };
553
554 // Insert prefetch op in init loop.
555 // Replace induction var with the init loop induction var.
556 rewriter.setInsertionPointToStart(initForOp.getBody());
557 xegpu::PrefetchNdOp::create(rewriter, newDescOp.getLoc(),
558 newDescOp.getResult(),
559 getPrefetchOffsets(initForOp.getInductionVar()),
560 readCacheHint, readCacheHint, readCacheHint,
561 /*layout=*/nullptr);
562
563 // Insert prefetch op in main loop.
564 // Calculate prefetch offset after the init prefetches have been issued.
565 rewriter.setInsertionPointToStart(forOp.getBody());
566 auto prefetchOffset = arith::AddIOp::create(rewriter, forOp.getLoc(),
567 forOp.getInductionVar(), nbStep);
568 // Replace induction var with correct offset.
569 xegpu::PrefetchNdOp::create(rewriter, newDescOp.getLoc(),
570 newDescOp.getResult(),
571 getPrefetchOffsets(prefetchOffset), readCacheHint,
572 readCacheHint, readCacheHint, /*layout=*/nullptr);
573
574 // Unroll the init loop.
575 if (failed(loopUnrollFull(initForOp)))
576 return emitSilenceableFailure(getLoc()) << "Failed to unroll the loop";
577
578 results.set(llvm::cast<OpResult>(getResult()), {newDescOp});
579
580 return DiagnosedSilenceableFailure::success();
581}
582
583void transform::InsertPrefetchOp::getEffects(
584 ::llvm::SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
585 onlyReadsHandle(getTargetMutable(), effects);
586 onlyReadsHandle(getDynamicNbPrefetchMutable(), effects);
587 producesHandle(getOperation()->getOpResults(), effects);
588 modifiesPayload(effects);
589}
590
591void transform::ConvertLayoutOp::build(
592 OpBuilder &builder, OperationState &ostate, Value target,
593 ArrayRef<OpFoldResult> mixedInputSgLayout,
594 ArrayRef<OpFoldResult> mixedInputSgData,
595 ArrayRef<OpFoldResult> mixedInputInstData, ArrayRef<int32_t> inputOrder,
596 ArrayRef<OpFoldResult> mixedTargetSgLayout,
597 ArrayRef<OpFoldResult> mixedTargetSgData,
598 ArrayRef<OpFoldResult> mixedTargetInstData, ArrayRef<int32_t> targetOrder) {
599 SmallVector<int64_t> staticInputSgLayout, staticInputSgData,
600 staticInputInstData;
601 SmallVector<Value> dynamicInputSgLayout, dynamicInputSgData,
602 dynamicInputInstData;
603 dispatchIndexOpFoldResults(ofrs: mixedInputSgLayout, dynamicVec&: dynamicInputSgLayout,
604 staticVec&: staticInputSgLayout);
605 dispatchIndexOpFoldResults(ofrs: mixedInputSgData, dynamicVec&: dynamicInputSgData,
606 staticVec&: staticInputSgData);
607 dispatchIndexOpFoldResults(ofrs: mixedInputInstData, dynamicVec&: dynamicInputInstData,
608 staticVec&: staticInputInstData);
609 SmallVector<int64_t> staticTargetSgLayout, staticTargetSgData,
610 staticTargetInstData;
611 SmallVector<Value> dynamicTargetSgLayout, dynamicTargetSgData,
612 dynamicTargetInstData;
613 dispatchIndexOpFoldResults(ofrs: mixedTargetSgLayout, dynamicVec&: dynamicTargetSgLayout,
614 staticVec&: staticTargetSgLayout);
615 dispatchIndexOpFoldResults(ofrs: mixedTargetSgData, dynamicVec&: dynamicTargetSgData,
616 staticVec&: staticTargetSgData);
617 dispatchIndexOpFoldResults(ofrs: mixedTargetInstData, dynamicVec&: dynamicTargetInstData,
618 staticVec&: staticTargetInstData);
619 build(builder, ostate, target.getType(),
620 /*target=*/target,
621 /*input_sg_layout=*/dynamicInputSgLayout,
622 /*input_sg_data=*/dynamicInputSgData,
623 /*input_inst_data=*/dynamicInputInstData,
624 /*target_sg_layout=*/dynamicTargetSgLayout,
625 /*target_sg_data=*/dynamicTargetSgData,
626 /*target_inst_data=*/dynamicTargetInstData,
627 /*input_order=*/inputOrder,
628 /*static_input_sg_layout=*/staticInputSgLayout,
629 /*static_input_sg_data=*/staticInputSgData,
630 /*static_input_inst_data=*/staticInputInstData,
631 /*static_target_sg_layout=*/staticTargetSgLayout,
632 /*static_target_sg_data=*/staticTargetSgData,
633 /*static_target_inst_data=*/staticTargetInstData,
634 /*target_order=*/targetOrder);
635}
636
637DiagnosedSilenceableFailure
638transform::ConvertLayoutOp::apply(transform::TransformRewriter &rewriter,
639 transform::TransformResults &results,
640 transform::TransformState &state) {
641 auto targetValues = state.getPayloadValues(handleValue: getTarget());
642 if (!llvm::hasSingleElement(targetValues))
643 return emitDefiniteFailure()
644 << "requires exactly one target value handle (got "
645 << llvm::range_size(targetValues) << ")";
646 auto value = *targetValues.begin();
647
648 // Construct layout attributes.
649 xegpu::LayoutAttr inputLayoutAttr = nullptr;
650 auto status = getLayoutAttrFromOperands(
651 getContext(), state, (*this), getMixedInputSgLayout(),
652 getMixedInputSgData(), getMixedInputInstData(),
653 /*packedSgLayout=*/Value(),
654 /*packedSgData=*/Value(), /*packedInstData=*/Value(), getInputOrder(),
655 inputLayoutAttr);
656 if (!status.succeeded())
657 return status;
658
659 xegpu::LayoutAttr targetLayoutAttr = nullptr;
660 status = getLayoutAttrFromOperands(
661 getContext(), state, (*this), getMixedTargetSgLayout(),
662 getMixedTargetSgData(), getMixedTargetInstData(),
663 /*packedSgLayout=*/Value(),
664 /*packedSgData=*/Value(), /*packedInstData=*/Value(), getTargetOrder(),
665 targetLayoutAttr);
666 if (!status.succeeded())
667 return status;
668
669 // Find first user op to define insertion point for layout conversion.
670 if (value.use_empty())
671 return emitSilenceableFailure(loc: getLoc())
672 << "Value has no users to insert layout conversion.";
673 Operation *userOp = *value.getUsers().begin();
674
675 // Emit convert_layout op.
676 rewriter.setInsertionPoint(userOp);
677 auto convLayoutOp =
678 xegpu::ConvertLayoutOp::create(rewriter, value.getLoc(), value.getType(),
679 value, inputLayoutAttr, targetLayoutAttr);
680 // Replace load op result with the converted layout.
681 rewriter.replaceUsesWithIf(
682 value, convLayoutOp.getResult(), [&](OpOperand &use) {
683 return use.getOwner() != convLayoutOp.getOperation();
684 });
685
686 results.set(llvm::cast<OpResult>(Val: getResult()), {convLayoutOp});
687 return DiagnosedSilenceableFailure::success();
688}
689
690void transform::ConvertLayoutOp::getEffects(
691 ::llvm::SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
692 onlyReadsHandle(handles: getTargetMutable(), effects);
693 onlyReadsHandle(getInputSgLayoutMutable(), effects);
694 onlyReadsHandle(getInputSgDataMutable(), effects);
695 onlyReadsHandle(getInputInstDataMutable(), effects);
696 onlyReadsHandle(getTargetSgLayoutMutable(), effects);
697 onlyReadsHandle(getTargetSgDataMutable(), effects);
698 onlyReadsHandle(getTargetInstDataMutable(), effects);
699 producesHandle(handles: getOperation()->getOpResults(), effects);
700 modifiesPayload(effects);
701}
702
703namespace {
704class XeGPUTransformDialectExtension
705 : public transform::TransformDialectExtension<
706 XeGPUTransformDialectExtension> {
707public:
708 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(XeGPUTransformDialectExtension)
709
710 using Base::Base;
711
712 void init();
713};
714
715void XeGPUTransformDialectExtension::init() {
716 declareGeneratedDialect<scf::SCFDialect>();
717 declareGeneratedDialect<arith::ArithDialect>();
718 declareGeneratedDialect<xegpu::XeGPUDialect>();
719
720 registerTransformOps<
721#define GET_OP_LIST
722#include "mlir/Dialect/XeGPU/TransformOps/XeGPUTransformOps.cpp.inc"
723 >();
724}
725} // namespace
726
727#define GET_OP_CLASSES
728#include "mlir/Dialect/XeGPU/TransformOps/XeGPUTransformOps.cpp.inc"
729
730void mlir::xegpu::registerTransformDialectExtension(DialectRegistry &registry) {
731 registry.addExtensions<XeGPUTransformDialectExtension>();
732}
733