1//===- ACCComputeLowering.cpp - Lower ACC compute to compute_region -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass decomposes OpenACC compute constructs into a representation that
10// separates the data environment from the compute portion and prepares for
11// parallelism assignment and privatization at the appropriate level.
12//
13// Overview:
14// ---------
15// Each compute construct (`acc.parallel`, `acc.serial`, `acc.kernels`) is
16// lowered to (1) `acc.kernel_environment`, which captures the data environment
17// and (2) `acc.compute_region`, which holds the compute body. Inside the
18// compute region, acc.loop is converted to SCF loops (`scf.parallel` or
19// `scf.for`) with any predetermined parallelism expressed as `par_dims`. This
20// decomposition allows later phases to assign parallelism and handle
21// privatization at the right granularity.
22//
23// Transformations:
24// ----------------
25// 1. Compute constructs: acc.parallel, acc.serial, and acc.kernels are
26// replaced by acc.kernel_environment containing a single acc.compute_region.
27// For acc.parallel / acc.kernels, launch arguments (num_gangs, num_workers,
28// vector_length) become acc.par_width ops (each result is `index`) and are
29// passed as compute_region launch operands. Compute regions with
30// num_gangs(1), num_workers(1), and vector_length(1) and acc serial use a
31// single sequential acc.par_width launch operand.
32//
33// 2. acc.loop: Converted according to context and attributes:
34// - Unstructured: body wrapped in scf.execute_region.
35// - Sequential (serial region, seq clause, or compute region with
36// num_gangs(1), num_workers(1), and vector_length(1)):
37// scf.parallel with par_dims = sequential.
38// - Auto (in parallel/kernels): scf.for with collapse when
39// multi-dimensional.
40// - Orphan (not inside a compute construct): scf.for, no collapse.
41// - Independent (in parallel/kernels): scf.parallel with par_dims from
42// gang/worker/vector mapping (e.g. block_x).
43//
44//===----------------------------------------------------------------------===//
45
46#include "mlir/Dialect/OpenACC/Extensions/AllExtensions.h"
47#include "mlir/Dialect/OpenACC/Transforms/Passes.h"
48
49#include "mlir/Dialect/Arith/IR/Arith.h"
50#include "mlir/Dialect/Func/IR/FuncOps.h"
51#include "mlir/Dialect/OpenACC/Analysis/OpenACCSupport.h"
52#include "mlir/Dialect/OpenACC/OpenACC.h"
53#include "mlir/Dialect/OpenACC/OpenACCParMapping.h"
54#include "mlir/Dialect/OpenACC/OpenACCUtils.h"
55#include "mlir/Dialect/OpenACC/OpenACCUtilsCG.h"
56#include "mlir/Dialect/OpenACC/OpenACCUtilsLoop.h"
57#include "mlir/Dialect/Utils/StaticValueUtils.h"
58#include "mlir/IR/IRMapping.h"
59#include "mlir/IR/Matchers.h"
60#include "mlir/Interfaces/FunctionInterfaces.h"
61#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
62#include "mlir/Transforms/RegionUtils.h"
63#include "llvm/ADT/DenseMap.h"
64#include "llvm/ADT/STLExtras.h"
65
66namespace mlir {
67namespace acc {
68#define GEN_PASS_DEF_ACCCOMPUTELOWERING
69#include "mlir/Dialect/OpenACC/Transforms/Passes.h.inc"
70} // namespace acc
71} // namespace mlir
72
73#define DEBUG_TYPE "acc-compute-lowering"
74
75using namespace mlir;
76using namespace mlir::acc;
77
78namespace {
79//===----------------------------------------------------------------------===//
80// Helper functions
81//===----------------------------------------------------------------------===//
82
83static bool isOpInComputeRegion(Operation *op) {
84 Region *region = op->getBlock()->getParent();
85 return getEnclosingComputeOp(region&: *region) != nullptr;
86}
87
88static bool isOpInSerialRegion(Operation *op) {
89 if (auto parallelOp = op->getParentOfType<ParallelOp>())
90 return parallelOp.isEffectivelySerial();
91 if (auto kernelsOp = op->getParentOfType<KernelsOp>())
92 return kernelsOp.isEffectivelySerial();
93 if (op->getParentOfType<SerialOp>())
94 return true;
95 if (auto computeRegion = op->getParentOfType<ComputeRegionOp>())
96 return computeRegion.isEffectivelySerial();
97 if (auto funcOp = op->getParentOfType<FunctionOpInterface>()) {
98 if (isSpecializedAccRoutine(op: funcOp)) {
99 auto attr = funcOp->getDiscardableAttrOfType<SpecializedRoutineAttr>(
100 name: getSpecializedRoutineAttrName());
101 if (attr && attr.getLevel().getValue() == ParLevel::seq)
102 return true;
103 }
104 }
105 return false;
106}
107
108/// Clone defining ops of constant live-in values into `region`, rewrite uses
109/// inside the region to the clones, and remove those values from
110/// `liveInValues` so they are not threaded through `acc.compute_region` ins.
111static void materializeConstantLiveInsIntoRegion(Region &region,
112 SetVector<Value> &liveInValues,
113 RewriterBase &rewriter) {
114 SmallVector<Value> constantLiveIns;
115 for (Value v : liveInValues) {
116 Operation *defOp = v.getDefiningOp();
117 if (defOp && matchPattern(op: defOp, pattern: m_Constant())) {
118 // As per the definition of ConstantLike trait, constants must have a
119 // single result.
120 assert(defOp->getNumResults() == 1 &&
121 "constants must have a single result");
122 constantLiveIns.push_back(Elt: v);
123 }
124 }
125 if (constantLiveIns.empty())
126 return;
127
128 OpBuilder::InsertionGuard guard(rewriter);
129 rewriter.setInsertionPointToStart(&region.front());
130
131 for (Value v : constantLiveIns) {
132 Value newV = rewriter.clone(op&: *v.getDefiningOp())->getResult(idx: 0);
133 replaceAllUsesInRegionWith(orig: v, replacement: newV, region);
134 liveInValues.remove(X: v);
135 }
136}
137
138/// Return the device type from which gang/worker/vector clauses should be read.
139/// If the requested device type has any such clauses, use that exclusively;
140/// otherwise fall back to the default (DeviceType::None).
141static DeviceType getGangWorkerVectorDeviceType(LoopOp loopOp,
142 DeviceType deviceType) {
143 if (deviceType != DeviceType::None &&
144 loopOp.hasAnyGangWorkerVector(deviceType))
145 return deviceType;
146 return DeviceType::None;
147}
148
149template <typename ComputeConstructT>
150static DeviceType getParDimsDeviceType(ComputeConstructT computeOp,
151 DeviceType deviceType) {
152 if (deviceType != DeviceType::None &&
153 computeOp.hasAnyGangWorkerVector(deviceType))
154 return deviceType;
155 return DeviceType::None;
156}
157
158/// Constant sized gang/worker/vector clauses collected per compute construct.
159struct SizedLevel {
160 ParLevel level;
161 int64_t size;
162};
163using SizedLevelMap = DenseMap<Operation *, SmallVector<SizedLevel>>;
164
165/// Record a sized clause if `size` is a constant. A non-constant vector or
166/// worker clause is ignored. Other non-constant sizes are NYI.
167static LogicalResult tryAddSizedLevel(SizedLevelMap &sizedLevelMap,
168 Operation *computeOp, LoopOp loopOp,
169 ParLevel level, Value size,
170 OpenACCSupport &accSupport) {
171 if (!size)
172 return success();
173 std::optional<int64_t> constSize = getConstantIntValue(ofr: size);
174 if (!constSize) {
175 if (level == ParLevel::vector || level == ParLevel::worker) {
176 accSupport.emitRemark(op: loopOp,
177 message: level == ParLevel::vector
178 ? "ignoring non-constant vector clause"
179 : "ignoring non-constant worker clause",
180 DEBUG_TYPE);
181 return success();
182 }
183 accSupport.emitNYI(loc: loopOp.getLoc(),
184 message: "non-constant sized parallelism clause");
185 return failure();
186 }
187 sizedLevelMap[computeOp].push_back(Elt: {.level: level, .size: *constSize});
188 return success();
189}
190
191/// Collect constant sized levels from loops in `acc.kernels` regions.
192static LogicalResult fillSizedLevelMap(Operation *op, DeviceType deviceType,
193 SizedLevelMap &sizedLevelMap,
194 OpenACCSupport &accSupport) {
195 WalkResult result = op->walk(callback: [&](LoopOp loopOp) {
196 Operation *computeOp =
197 getEnclosingComputeOp(region&: *loopOp->getBlock()->getParent());
198 if (!computeOp || !isa<KernelsOp>(Val: computeOp))
199 return WalkResult::advance();
200 DeviceType loopDeviceType =
201 getGangWorkerVectorDeviceType(loopOp, deviceType);
202 if (failed(Result: tryAddSizedLevel(
203 sizedLevelMap, computeOp, loopOp, level: ParLevel::vector,
204 size: loopOp.getVectorValue(deviceType: loopDeviceType), accSupport)) ||
205 failed(Result: tryAddSizedLevel(
206 sizedLevelMap, computeOp, loopOp, level: ParLevel::worker,
207 size: loopOp.getWorkerValue(deviceType: loopDeviceType), accSupport)) ||
208 failed(Result: tryAddSizedLevel(
209 sizedLevelMap, computeOp, loopOp, level: ParLevel::gang_dim1,
210 size: loopOp.getGangValue(gangArgType: GangArgType::Num, deviceType: loopDeviceType), accSupport)))
211 return WalkResult::interrupt();
212 return WalkResult::advance();
213 });
214 return failure(IsFailure: result.wasInterrupted());
215}
216
217/// Map loop parallelism clauses (gang/worker/vector) to GPU parallel
218/// dimensions using the given mapping policy. Sized clauses (e.g. vector(n))
219/// count as the corresponding level.
220static SmallVector<GPUParallelDimAttr>
221getParallelDimensions(LoopOp loopOp, const ACCToGPUMappingPolicy &policy,
222 DeviceType deviceType) {
223 deviceType = getGangWorkerVectorDeviceType(loopOp, deviceType);
224 SmallVector<GPUParallelDimAttr> parDims;
225 auto *ctx = loopOp->getContext();
226
227 if (loopOp.hasVector(deviceType) || loopOp.getVectorValue(deviceType))
228 insertParDim(parDims, parDim: policy.vectorDim(ctx));
229 if (loopOp.hasWorker(deviceType) || loopOp.getWorkerValue(deviceType))
230 insertParDim(parDims, parDim: policy.workerDim(ctx));
231 if (auto gangDimValue = loopOp.getGangValue(gangArgType: GangArgType::Dim, deviceType)) {
232 if (auto gangDimDefOp =
233 gangDimValue.getDefiningOp<arith::ConstantIntOp>()) {
234 auto gangLevel = getGangParLevel(gangDimValue: gangDimDefOp.value());
235 insertParDim(parDims, parDim: policy.gangDim(ctx, level: gangLevel));
236 }
237 } else if (loopOp.hasGang(deviceType) ||
238 loopOp.getGangValue(gangArgType: GangArgType::Num, deviceType) ||
239 loopOp.getGangValue(gangArgType: GangArgType::Static, deviceType)) {
240 insertParDim(parDims, parDim: policy.gangDim(ctx, level: ParLevel::gang_dim1));
241 }
242 return parDims;
243}
244
245/// If \p loopOp has a `gang(static:)` value, return the corresponding
246/// `acc.chunk_size` attribute. A constant size is stored directly; a
247/// non-constant size is recorded as -1, the same encoding as `static:*`.
248/// Returns a null attribute when the clause is absent.
249static ChunkSizeAttr getChunkSizeFromStatic(LoopOp loopOp,
250 DeviceType deviceType) {
251 DeviceType loopDeviceType = getGangWorkerVectorDeviceType(loopOp, deviceType);
252 Value gangStatic = loopOp.getGangValue(gangArgType: GangArgType::Static, deviceType: loopDeviceType);
253 if (!gangStatic)
254 return {};
255 std::optional<int64_t> chunkSize = getConstantIntValue(ofr: gangStatic);
256 // Non-constant static is treated like `static:*` (implementation-chosen).
257 return ChunkSizeAttr::get(context: loopOp->getContext(), chunk_size: chunkSize.value_or(u: -1));
258}
259
260/// Build `acc.compute_region` launch operands: one sequential `acc.par_width`
261/// for `acc.serial`, for `acc.parallel` / `acc.kernels` when every num_gangs
262/// operand and num_workers / vector_length are the constant 1, and otherwise
263/// `acc.par_width` from gang/worker/vector (device-type operands first, then
264/// default DeviceType::None).
265template <typename ComputeConstructT>
266static SmallVector<Value> assignKnownLaunchArgs(
267 ComputeConstructT computeOp, DeviceType deviceType, RewriterBase &rewriter,
268 const ACCToGPUMappingPolicy &policy, const SizedLevelMap &sizedLevelMap) {
269 auto *ctx = rewriter.getContext();
270 auto loc = computeOp->getLoc();
271
272 if constexpr (std::is_same_v<ComputeConstructT, SerialOp>) {
273 return {ParWidthOp::create(rewriter, loc, Value(), policy.seqDim(ctx))};
274 } else if constexpr (llvm::is_one_of<ComputeConstructT, ParallelOp,
275 KernelsOp>::value) {
276 if (computeOp.isEffectivelySerial())
277 return {ParWidthOp::create(rewriter, loc, Value(), policy.seqDim(ctx))};
278
279 deviceType = getParDimsDeviceType(computeOp, deviceType);
280
281 SmallVector<Value> values;
282 auto indexTy = rewriter.getIndexType();
283
284 auto numGangs = computeOp.getNumGangsValues(deviceType);
285 for (auto [gangDimIdx, gangSize] : llvm::enumerate(numGangs)) {
286 auto gangLevel = getGangParLevel(gangDimIdx + 1);
287 values.push_back(Elt: ParWidthOp::create(
288 rewriter, loc,
289 getValueOrCreateCastToIndexLike(rewriter, gangSize.getLoc(), indexTy,
290 gangSize),
291 policy.gangDim(ctx, level: gangLevel)));
292 }
293
294 Value numWorkers = computeOp.getNumWorkersValue(deviceType);
295 if (numWorkers) {
296 values.push_back(Elt: ParWidthOp::create(
297 rewriter, loc,
298 getValueOrCreateCastToIndexLike(b&: rewriter, loc: numWorkers.getLoc(),
299 targetType: indexTy, value: numWorkers),
300 policy.workerDim(ctx)));
301 }
302
303 Value vectorLength = computeOp.getVectorLengthValue(deviceType);
304 if (vectorLength) {
305 values.push_back(Elt: ParWidthOp::create(
306 rewriter, loc,
307 getValueOrCreateCastToIndexLike(b&: rewriter, loc: vectorLength.getLoc(),
308 targetType: indexTy, value: vectorLength),
309 policy.vectorDim(ctx)));
310 }
311
312 // Loop-level sized clauses. Skip a dim already set on the construct.
313 // Rematerialize the constant here so it dominates the compute region.
314 auto sizedLevels = sizedLevelMap.find(computeOp.getOperation());
315 if (sizedLevels != sizedLevelMap.end()) {
316 for (const SizedLevel &sizedLevel : sizedLevels->second) {
317 GPUParallelDimAttr dim = policy.map(ctx, level: sizedLevel.level);
318 bool exists = llvm::any_of(values, [&](Value v) {
319 auto parWidth = v.getDefiningOp<ParWidthOp>();
320 return parWidth && parWidth.getParDim() == dim;
321 });
322 if (exists)
323 continue;
324 Value sizeVal =
325 arith::ConstantIndexOp::create(rewriter, loc, sizedLevel.size);
326 values.push_back(Elt: ParWidthOp::create(rewriter, loc, sizeVal, dim));
327 }
328 }
329 return values;
330 } else {
331 llvm_unreachable("assignKnownLaunchArgs: expected parallel, kernels, or "
332 "serial");
333 }
334}
335
336//===----------------------------------------------------------------------===//
337// Loop conversion pattern
338//===----------------------------------------------------------------------===//
339
340class ACCLoopConversion : public OpRewritePattern<LoopOp> {
341public:
342 ACCLoopConversion(MLIRContext *ctx, const ACCToGPUMappingPolicy &policy,
343 DeviceType deviceType)
344 : OpRewritePattern<LoopOp>(ctx), policy(policy), deviceType(deviceType) {}
345
346 LogicalResult matchAndRewrite(LoopOp loopOp,
347 PatternRewriter &rewriter) const override {
348 if (loopOp.getUnstructured()) {
349 // An independent loop is asserted parallel but cannot be parallelized
350 // once it is lowered as a region of blocks. Record that, so the loss
351 // can be reported instead of silently running the loop sequentially.
352 bool lostIndependence =
353 loopOp.getDefaultOrDeviceTypeParallelism(deviceType) ==
354 LoopParMode::loop_independent &&
355 !isOpInSerialRegion(op: loopOp) &&
356 (isOpInComputeRegion(op: loopOp) ||
357 isSpecializedAccRoutine(
358 op: loopOp->getParentOfType<FunctionOpInterface>()));
359 auto executeRegion =
360 convertUnstructuredACCLoopToSCFExecuteRegion(loopOp, rewriter);
361 if (!executeRegion)
362 return failure();
363 if (lostIndependence)
364 executeRegion->setDiscardableAttr(
365 name: getUnstructuredIndependentLoopAttrName(), value: rewriter.getUnitAttr());
366 rewriter.replaceOp(op: loopOp, newOp: executeRegion);
367 return success();
368 }
369
370 LoopParMode parMode = loopOp.getDefaultOrDeviceTypeParallelism(deviceType);
371
372 if (parMode == LoopParMode::loop_seq || isOpInSerialRegion(op: loopOp)) {
373 // Use scf.for with sequential loops, because the loop's parallelism is
374 // already determined.
375 auto forOp =
376 convertACCLoopToSCFFor(loopOp, rewriter, /*enableCollapse=*/true);
377 if (!forOp)
378 return failure();
379 setParDimsAttr(op: forOp, attr: GPUParallelDimsAttr::seq(loopOp->getContext()));
380 rewriter.replaceOp(op: loopOp, newOp: forOp);
381 } else if (parMode == LoopParMode::loop_auto) {
382 // All loops in serial regions should have already been handled.
383 assert(!isOpInSerialRegion(loopOp) &&
384 "Expected loop to be in non-serial region");
385 // Mark as scf.for to allow auto-parallelization analysis later.
386 auto forOp =
387 convertACCLoopToSCFFor(loopOp, rewriter, /*enableCollapse=*/true);
388 if (!forOp)
389 return failure();
390 SmallVector<GPUParallelDimAttr> parDims =
391 getParallelDimensions(loopOp, policy, deviceType);
392 if (!parDims.empty()) {
393 auto parDimsAttr =
394 GPUParallelDimsAttr::get(context: loopOp->getContext(), array: parDims);
395 setParDimsAttr(op: forOp, attr: parDimsAttr);
396 if (ChunkSizeAttr chunkSize =
397 getChunkSizeFromStatic(loopOp, deviceType))
398 setChunkSizeAttr(op: forOp, attr: chunkSize);
399 }
400 rewriter.replaceOp(op: loopOp, newOp: forOp);
401 } else if (!isOpInComputeRegion(op: loopOp) &&
402 !isSpecializedAccRoutine(
403 op: loopOp->getParentOfType<FunctionOpInterface>())) {
404 // This loop is an orphan `acc loop` but it is not in any sort
405 // of compute region. Thus it is just a sequential non-accelerator loop.
406 auto forOp =
407 convertACCLoopToSCFFor(loopOp, rewriter, /*enableCollapse=*/false);
408 if (!forOp)
409 return failure();
410 rewriter.replaceOp(op: loopOp, newOp: forOp);
411 } else {
412 assert(parMode == LoopParMode::loop_independent &&
413 "Expected loop to be independent");
414 auto parallelOp = convertACCLoopToSCFParallel(loopOp, rewriter);
415 if (!parallelOp)
416 return failure();
417
418 SmallVector<GPUParallelDimAttr> parDims =
419 getParallelDimensions(loopOp, policy, deviceType);
420 if (!parDims.empty()) {
421 auto parDimsAttr =
422 GPUParallelDimsAttr::get(context: loopOp->getContext(), array: parDims);
423 setParDimsAttr(op: parallelOp, attr: parDimsAttr);
424 if (ChunkSizeAttr chunkSize =
425 getChunkSizeFromStatic(loopOp, deviceType))
426 setChunkSizeAttr(op: parallelOp, attr: chunkSize);
427 }
428 rewriter.replaceOp(op: loopOp, newOp: parallelOp);
429 }
430 return success();
431 }
432
433private:
434 const ACCToGPUMappingPolicy &policy;
435 DeviceType deviceType;
436};
437
438//===----------------------------------------------------------------------===//
439// Compute construct conversion pattern
440//===----------------------------------------------------------------------===//
441
442template <typename ComputeConstructT>
443class ComputeOpConversion : public OpRewritePattern<ComputeConstructT> {
444public:
445 ComputeOpConversion(MLIRContext *ctx, const ACCToGPUMappingPolicy &policy,
446 DeviceType deviceType, const SizedLevelMap &sizedLevelMap)
447 : OpRewritePattern<ComputeConstructT>(ctx), policy(policy),
448 deviceType(deviceType), sizedLevelMap(sizedLevelMap) {}
449
450 LogicalResult matchAndRewrite(ComputeConstructT computeOp,
451 PatternRewriter &rewriter) const override {
452 rewriter.setInsertionPoint(computeOp);
453 auto kernelEnv =
454 KernelEnvironmentOp::createAndPopulate(computeOp, deviceType, rewriter);
455 auto launchArgs = assignKnownLaunchArgs(computeOp, deviceType, rewriter,
456 policy, sizedLevelMap);
457 Region &region = computeOp.getRegion();
458 SetVector<Value> liveInValues;
459 getUsedValuesDefinedAbove(region, limit&: region, values&: liveInValues);
460 materializeConstantLiveInsIntoRegion(region, liveInValues, rewriter);
461 IRMapping mapping;
462 auto computeRegion = buildComputeRegion(
463 computeOp->getLoc(), launchArgs, liveInValues.getArrayRef(),
464 ComputeConstructT::getOperationName(), region, rewriter, mapping);
465 if (!computeRegion) {
466 rewriter.eraseOp(op: kernelEnv);
467 return failure();
468 }
469 rewriter.eraseOp(op: computeOp);
470 return success();
471 }
472
473private:
474 const ACCToGPUMappingPolicy &policy;
475 DeviceType deviceType;
476 const SizedLevelMap &sizedLevelMap;
477};
478
479//===----------------------------------------------------------------------===//
480// Pass implementation
481//===----------------------------------------------------------------------===//
482
483class ACCComputeLowering
484 : public acc::impl::ACCComputeLoweringBase<ACCComputeLowering> {
485public:
486 using ACCComputeLoweringBase::ACCComputeLoweringBase;
487
488 void getDependentDialects(DialectRegistry &registry) const override {
489 ACCComputeLoweringBase::getDependentDialects(registry);
490 acc::registerAllExtensions(registry);
491 }
492
493 void runOnOperation() override {
494 auto op = getOperation();
495 auto *context = op.getContext();
496
497 DefaultACCToGPUMappingPolicy policy;
498 // Collect loop sized levels before loops are rewritten away.
499 SizedLevelMap sizedLevelMap;
500 OpenACCSupport &accSupport = getAnalysis<OpenACCSupport>();
501 if (failed(Result: fillSizedLevelMap(op, deviceType, sizedLevelMap, accSupport)))
502 return signalPassFailure();
503
504 // Part 1: Convert acc.loop to scf.parallel/scf.for while the parent
505 // compute construct is still present (needed to determine conversion
506 // strategy).
507 RewritePatternSet loopPatterns(context);
508 loopPatterns.insert<ACCLoopConversion>(arg&: context, args&: policy, args&: deviceType);
509 if (failed(Result: applyPatternsGreedily(op, patterns: std::move(loopPatterns))))
510 return signalPassFailure();
511
512 // Part 2: Convert acc.parallel, acc.kernels, and acc.serial to
513 // acc.kernel_environment { acc.compute_region { ... } }.
514 RewritePatternSet computePatterns(context);
515 computePatterns
516 .insert<ComputeOpConversion<ParallelOp>, ComputeOpConversion<KernelsOp>,
517 ComputeOpConversion<SerialOp>>(arg&: context, args&: policy, args&: deviceType,
518 args&: sizedLevelMap);
519 if (failed(Result: applyPatternsGreedily(op, patterns: std::move(computePatterns))))
520 return signalPassFailure();
521 }
522};
523} // namespace
524