1//===- OpenACCCG.cpp - OpenACC codegen ops, attributes, and types ---------===//
2//
3// Part of the MLIR Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Implementation for OpenACC codegen operations, attributes, and types.
10// These correspond to the definitions in OpenACCCG*.td tablegen files
11// and are kept in a separate file because they do not represent direct mappings
12// of OpenACC language constructs; they are intermediate representations used
13// when decomposing and lowering primary `acc` dialect operations.
14//
15//===----------------------------------------------------------------------===//
16
17#include "mlir/Dialect/Arith/IR/Arith.h"
18#include "mlir/Dialect/GPU/IR/GPUBase.h"
19#include "mlir/Dialect/OpenACC/OpenACC.h"
20#include "mlir/Dialect/Utils/StaticValueUtils.h"
21#include "mlir/IR/BuiltinAttributes.h"
22#include "mlir/IR/BuiltinTypes.h"
23#include "mlir/IR/PatternMatch.h"
24#include "mlir/IR/Region.h"
25#include "mlir/Interfaces/ControlFlowInterfaces.h"
26#include "mlir/Support/LogicalResult.h"
27#include "mlir/Transforms/RegionUtils.h"
28#include "llvm/ADT/DenseMap.h"
29#include "llvm/ADT/STLExtras.h"
30#include "llvm/ADT/SmallVector.h"
31
32using namespace mlir;
33using namespace acc;
34
35namespace {
36/// Generic helper for single-region OpenACC ops that execute their body once
37/// and then continue after the operation with their results (if any).
38static void
39getSingleRegionOpSuccessorRegions(Operation *op, Region &region,
40 RegionBranchPoint point,
41 SmallVectorImpl<RegionSuccessor> &regions) {
42 if (point.isParent()) {
43 regions.push_back(Elt: RegionSuccessor(&region));
44 return;
45 }
46 regions.push_back(Elt: RegionSuccessor(op));
47}
48
49static ValueRange getSingleRegionSuccessorInputs(Operation *op,
50 RegionSuccessor successor) {
51 return successor.isOperation() ? ValueRange(op->getResults()) : ValueRange();
52}
53
54/// Remove empty acc.kernel_environment operations. If the operation has wait
55/// operands, create a acc.wait operation to preserve synchronization.
56struct RemoveEmptyKernelEnvironment
57 : public OpRewritePattern<acc::KernelEnvironmentOp> {
58 using OpRewritePattern<acc::KernelEnvironmentOp>::OpRewritePattern;
59
60 LogicalResult matchAndRewrite(acc::KernelEnvironmentOp op,
61 PatternRewriter &rewriter) const override {
62 assert(op->getNumRegions() == 1 && "expected op to have one region");
63
64 Block &block = op.getRegion().front();
65 if (!block.empty())
66 return failure();
67
68 // Remove empty kernel environment.
69 // Preserve synchronization by creating acc.wait operation if needed.
70 if (!op.getWaitOperands().empty() || op.getWaitOnlyAttr())
71 rewriter.replaceOpWithNewOp<acc::WaitOp>(
72 op, args: op.getWaitOperands(), /*asyncOperand=*/args: Value(),
73 args: op.getWaitDevnum(), /*async=*/args: nullptr, /*ifCond=*/args: Value());
74 else
75 rewriter.eraseOp(op);
76
77 return success();
78 }
79};
80
81struct KernelEnvironmentDedupDataOperands
82 : public OpRewritePattern<acc::KernelEnvironmentOp> {
83 using OpRewritePattern<acc::KernelEnvironmentOp>::OpRewritePattern;
84
85 static bool isReductionDataOperand(Value operand) {
86 Operation *def = operand.getDefiningOp();
87 if (!def)
88 return false;
89 std::optional<DataClause> clause = getDataClause(accDataEntryOp: def);
90 if (!clause || *clause != DataClause::acc_reduction)
91 return false;
92 assert(isa<CopyinOp>(def) && "reduction clause must be an acc.copyin");
93 return true;
94 }
95
96 LogicalResult matchAndRewrite(acc::KernelEnvironmentOp op,
97 PatternRewriter &rewriter) const override {
98 SmallVector<Value> operands(op.getDataClauseOperands());
99 const unsigned numOperands = operands.size();
100 if (numOperands < 2)
101 return failure();
102
103 SmallVector<Value> kept;
104 SmallVector<std::pair<Value, Value>> useRewrites;
105 SmallVector<SmallVector<Value>> boundsStorage;
106 boundsStorage.reserve(N: numOperands);
107 llvm::DenseMap<std::pair<Value, ArrayRef<Value>>, unsigned>
108 keptIndexByVarAndBounds;
109 for (Value operand : operands) {
110 Operation *def = operand.getDefiningOp();
111 // acc.map_info is a finalized mapping. Its order and semantics stand;
112 // matching only its variable and bounds would ignore flags and
113 // descriptors.
114 if (!def || isa<MapInfoOp>(Val: def)) {
115 kept.push_back(Elt: operand);
116 continue;
117 }
118
119 Value var = getVar(accDataClauseOp: def);
120 if (!var) {
121 kept.push_back(Elt: operand);
122 continue;
123 }
124
125 boundsStorage.push_back(Elt: getBounds(accDataClauseOp: def));
126 auto [it, inserted] = keptIndexByVarAndBounds.try_emplace(
127 Key: std::make_pair(x&: var, y: ArrayRef<Value>(boundsStorage.back())),
128 Args: kept.size());
129 if (inserted) {
130 kept.push_back(Elt: operand);
131 continue;
132 }
133
134 Value &survivor = kept[it->second];
135 // A reduction acc.copyin is that copyin plus the reduction. Keeping
136 // another clause would drop the reduction.
137 if (isReductionDataOperand(operand) && !isReductionDataOperand(operand: survivor))
138 std::swap(a&: survivor, b&: operand);
139 useRewrites.emplace_back(Args&: operand, Args&: survivor);
140 }
141 if (kept.size() == numOperands)
142 return failure();
143
144 for (auto [from, to] : useRewrites) {
145 rewriter.replaceUsesWithIf(from, to, functor: [&](OpOperand &use) {
146 return op->isProperAncestor(other: use.getOwner());
147 });
148 }
149 rewriter.modifyOpInPlace(
150 root: op, callable: [&] { op.getDataClauseOperandsMutable().assign(values: kept); });
151 return success();
152 }
153};
154
155// Capture `hasStream` before erasing inputs because segment metadata is only
156// repaired at the end of the modification transaction.
157static void setComputeRegionInputOperandSegments(ComputeRegionOp op,
158 PatternRewriter &rewriter,
159 size_t numInput,
160 bool hasStream) {
161 const size_t numLaunch = op.getLaunchArgs().size();
162 op->setInherentAttr(
163 name: rewriter.getStringAttr(bytes: ComputeRegionOp::getOperandSegmentSizeAttr()),
164 value: rewriter.getDenseI32ArrayAttr(values: {static_cast<int32_t>(numLaunch),
165 static_cast<int32_t>(numInput),
166 hasStream ? 1 : 0}));
167}
168
169struct ComputeRegionRemoveDuplicateArgs
170 : public OpRewritePattern<ComputeRegionOp> {
171 using Base::Base;
172
173 LogicalResult matchAndRewrite(ComputeRegionOp op,
174 PatternRewriter &rewriter) const override {
175 Block *body = op.getBody();
176 const size_t numLaunch = op.getLaunchArgs().size();
177 size_t numInput = op.getInputArgs().size();
178 assert(body->getNumArguments() == numLaunch + numInput &&
179 "region args mismatch");
180
181 bool hasDuplicate = false;
182 for (size_t j = 1; j < numInput && !hasDuplicate; ++j)
183 for (size_t i = 0; i < j; ++i)
184 if (op->getOperand(idx: static_cast<unsigned>(numLaunch + i)) ==
185 op->getOperand(idx: static_cast<unsigned>(numLaunch + j))) {
186 hasDuplicate = true;
187 break;
188 }
189 if (!hasDuplicate)
190 return failure();
191
192 const bool hasStream = static_cast<bool>(op.getStream());
193 rewriter.modifyOpInPlace(root: op, callable: [&] {
194 while (true) {
195 bool merged = false;
196 for (size_t j = 1; j < numInput && !merged; ++j) {
197 for (size_t i = 0; i < j; ++i) {
198 if (op->getOperand(idx: static_cast<unsigned>(numLaunch + i)) !=
199 op->getOperand(idx: static_cast<unsigned>(numLaunch + j)))
200 continue;
201 unsigned keepIdx = static_cast<unsigned>(numLaunch + i);
202 unsigned dropIdx = static_cast<unsigned>(numLaunch + j);
203 rewriter.replaceAllUsesWith(from: body->getArgument(i: dropIdx),
204 to: body->getArgument(i: keepIdx));
205 body->eraseArgument(index: dropIdx);
206 op->eraseOperand(idx: dropIdx);
207 --numInput;
208 merged = true;
209 break;
210 }
211 }
212 if (!merged)
213 break;
214 }
215 setComputeRegionInputOperandSegments(op, rewriter, numInput, hasStream);
216 });
217 return success();
218 }
219};
220
221struct ComputeRegionRemoveUnusedArgs
222 : public OpRewritePattern<ComputeRegionOp> {
223 using Base::Base;
224
225 LogicalResult matchAndRewrite(ComputeRegionOp op,
226 PatternRewriter &rewriter) const override {
227 Block *body = op.getBody();
228 const size_t numLaunch = op.getLaunchArgs().size();
229 size_t numInput = op.getInputArgs().size();
230 assert(body->getNumArguments() == numLaunch + numInput &&
231 "region args mismatch");
232
233 bool hasUnused = false;
234 for (size_t k = numLaunch; k < numLaunch + numInput; ++k)
235 if (body->getArgument(i: static_cast<unsigned>(k)).use_empty()) {
236 hasUnused = true;
237 break;
238 }
239 if (!hasUnused)
240 return failure();
241
242 const bool hasStream = static_cast<bool>(op.getStream());
243 rewriter.modifyOpInPlace(root: op, callable: [&] {
244 for (size_t k = numLaunch; k < numLaunch + numInput;) {
245 if (!body->getArgument(i: static_cast<unsigned>(k)).use_empty()) {
246 ++k;
247 continue;
248 }
249 body->eraseArgument(index: static_cast<unsigned>(k));
250 op->eraseOperand(idx: static_cast<unsigned>(k));
251 --numInput;
252 }
253 setComputeRegionInputOperandSegments(op, rewriter, numInput, hasStream);
254 });
255 return success();
256 }
257};
258
259template <typename EffectTy>
260static void addOperandEffect(
261 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
262 &effects,
263 const MutableOperandRange &operand) {
264 for (unsigned i = 0, e = operand.size(); i < e; ++i)
265 effects.emplace_back(EffectTy::get(), &operand[i]);
266}
267
268template <typename EffectTy>
269static void addResultEffect(
270 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
271 &effects,
272 Value result) {
273 effects.emplace_back(EffectTy::get(), mlir::cast<mlir::OpResult>(Val&: result));
274}
275
276static int64_t gpuProcessorIndex(gpu::Processor p) {
277 switch (p) {
278 case gpu::Processor::Sequential:
279 return 0;
280 case gpu::Processor::ThreadX:
281 return 1;
282 case gpu::Processor::ThreadY:
283 return 2;
284 case gpu::Processor::ThreadZ:
285 return 3;
286 case gpu::Processor::BlockX:
287 return 4;
288 case gpu::Processor::BlockY:
289 return 5;
290 case gpu::Processor::BlockZ:
291 return 6;
292 }
293 llvm_unreachable("unhandled gpu::Processor");
294}
295
296static gpu::Processor indexToGpuProcessor(int64_t idx) {
297 switch (idx) {
298 case 0:
299 return gpu::Processor::Sequential;
300 case 1:
301 return gpu::Processor::ThreadX;
302 case 2:
303 return gpu::Processor::ThreadY;
304 case 3:
305 return gpu::Processor::ThreadZ;
306 case 4:
307 return gpu::Processor::BlockX;
308 case 5:
309 return gpu::Processor::BlockY;
310 case 6:
311 return gpu::Processor::BlockZ;
312 default:
313 return gpu::Processor::Sequential;
314 }
315}
316
317static GPUParallelDimAttr intToParDim(MLIRContext *context, int64_t dimInt) {
318 return GPUParallelDimAttr::get(
319 context, value: IntegerAttr::get(type: IndexType::get(context), value: dimInt));
320}
321
322static GPUParallelDimAttr processorParDim(MLIRContext *context,
323 gpu::Processor proc) {
324 return GPUParallelDimAttr::get(
325 context,
326 value: IntegerAttr::get(type: IndexType::get(context), value: gpuProcessorIndex(p: proc)));
327}
328
329static ParseResult parseProcessorValue(AsmParser &parser,
330 GPUParallelDimAttr &dim) {
331 std::string keyword;
332 llvm::SMLoc loc = parser.getCurrentLocation();
333 if (failed(Result: parser.parseKeywordOrString(result: &keyword)))
334 return failure();
335 auto maybeProcessor = gpu::symbolizeProcessor(keyword);
336 if (!maybeProcessor)
337 return parser.emitError(loc)
338 << "expected one of ::mlir::gpu::Processor enum names";
339 dim = intToParDim(context: parser.getContext(), dimInt: gpuProcessorIndex(p: *maybeProcessor));
340 return success();
341}
342
343static void printProcessorValue(AsmPrinter &printer,
344 const GPUParallelDimAttr &attr) {
345 gpu::Processor processor = indexToGpuProcessor(idx: attr.getValue().getInt());
346 printer << gpu::stringifyProcessor(processor);
347}
348
349static FailureOr<SmallVector<GPUParallelDimAttr>>
350parseGPUParallelDimList(AsmParser &parser) {
351 SmallVector<GPUParallelDimAttr> parDims;
352 auto parseParDim = [&]() -> ParseResult {
353 GPUParallelDimAttr dim;
354 if (parseProcessorValue(parser, dim))
355 return failure();
356 parDims.push_back(Elt: dim);
357 return success();
358 };
359 if (parser.parseCommaSeparatedList(delimiter: AsmParser::Delimiter::Square, parseElementFn: parseParDim,
360 contextMessage: "list of OpenACC GPU parallel dimensions"))
361 return failure();
362 return parDims;
363}
364
365static void printGPUParallelDimList(AsmPrinter &printer,
366 ArrayRef<GPUParallelDimAttr> dims) {
367 printer << "[";
368 llvm::interleaveComma(c: dims, os&: printer, each_fn: [&printer](const GPUParallelDimAttr &p) {
369 printProcessorValue(printer, attr: p);
370 });
371 printer << "]";
372}
373} // namespace
374
375//===----------------------------------------------------------------------===//
376// KernelEnvironmentOp
377//===----------------------------------------------------------------------===//
378
379void KernelEnvironmentOp::getSuccessorRegions(
380 RegionBranchPoint point, SmallVectorImpl<RegionSuccessor> &regions) {
381 getSingleRegionOpSuccessorRegions(op: getOperation(), region&: getRegion(), point,
382 regions);
383}
384
385ValueRange KernelEnvironmentOp::getSuccessorInputs(RegionSuccessor successor) {
386 return getSingleRegionSuccessorInputs(op: getOperation(), successor);
387}
388
389void KernelEnvironmentOp::getCanonicalizationPatterns(
390 RewritePatternSet &results, MLIRContext *context) {
391 results.add<RemoveEmptyKernelEnvironment, KernelEnvironmentDedupDataOperands>(
392 arg&: context);
393}
394
395/// Extract async for `clauseDeviceType`. Returns true if a clause was found.
396template <typename ComputeConstructT>
397static bool
398extractAsyncClause(ComputeConstructT computeConstruct,
399 DeviceType clauseDeviceType, MLIRContext *context,
400 std::optional<Value> &asyncOperand, UnitAttr &asyncOnly) {
401 if (computeConstruct.hasAsyncOnly(clauseDeviceType)) {
402 asyncOnly = UnitAttr::get(context);
403 return true;
404 }
405 if (Value asyncValue = computeConstruct.getAsyncValue(clauseDeviceType)) {
406 asyncOperand = asyncValue;
407 return true;
408 }
409 return false;
410}
411
412/// Extract wait for `clauseDeviceType`. Returns true if a clause was found.
413template <typename ComputeConstructT>
414static bool extractWaitClause(ComputeConstructT computeConstruct,
415 DeviceType clauseDeviceType, MLIRContext *context,
416 std::optional<Value> &waitDevnum,
417 SmallVectorImpl<Value> &waitOperands,
418 UnitAttr &waitOnly) {
419 if (computeConstruct.hasWaitOnly(clauseDeviceType)) {
420 waitOnly = UnitAttr::get(context);
421 return true;
422 }
423 Value devnum = computeConstruct.getWaitDevnum(clauseDeviceType);
424 auto waitValues = computeConstruct.getWaitValues(clauseDeviceType);
425 if (!devnum && waitValues.empty())
426 return false;
427 if (devnum)
428 waitDevnum = devnum;
429 waitOperands.append(waitValues.begin(), waitValues.end());
430 return true;
431}
432
433template <typename ComputeConstructT>
434static void populateKernelEnvironmentAsyncWait(
435 ComputeConstructT computeConstruct, DeviceType deviceType,
436 std::optional<Value> &asyncOperand, UnitAttr &asyncOnly,
437 std::optional<Value> &waitDevnum, SmallVectorImpl<Value> &waitOperands,
438 UnitAttr &waitOnly) {
439 MLIRContext *context = computeConstruct->getContext();
440
441 // Prefer device_type-specific clauses, then default ones.
442 if (!extractAsyncClause(computeConstruct, deviceType, context, asyncOperand,
443 asyncOnly)) {
444 if (deviceType != DeviceType::None)
445 extractAsyncClause(computeConstruct, DeviceType::None, context,
446 asyncOperand, asyncOnly);
447 }
448
449 if (!extractWaitClause(computeConstruct, deviceType, context, waitDevnum,
450 waitOperands, waitOnly)) {
451 if (deviceType != DeviceType::None)
452 extractWaitClause(computeConstruct, DeviceType::None, context, waitDevnum,
453 waitOperands, waitOnly);
454 }
455}
456
457template <typename ComputeConstructT>
458KernelEnvironmentOp
459KernelEnvironmentOp::createAndPopulate(ComputeConstructT computeConstruct,
460 DeviceType deviceType,
461 OpBuilder &builder) {
462 std::optional<Value> asyncOperand;
463 UnitAttr asyncOnly = nullptr;
464 std::optional<Value> waitDevnum;
465 SmallVector<Value> waitOperands;
466 UnitAttr waitOnly = nullptr;
467 populateKernelEnvironmentAsyncWait(computeConstruct, deviceType, asyncOperand,
468 asyncOnly, waitDevnum, waitOperands,
469 waitOnly);
470
471 auto kernelEnvironment = KernelEnvironmentOp::create(
472 builder, computeConstruct->getLoc(),
473 computeConstruct.getDataClauseOperands(), asyncOperand.value_or(u: Value()),
474 asyncOnly, waitDevnum.value_or(u: Value()), waitOperands, waitOnly);
475 Block &block = kernelEnvironment.getRegion().emplaceBlock();
476 builder.setInsertionPointToStart(&block);
477 return kernelEnvironment;
478}
479
480template KernelEnvironmentOp
481KernelEnvironmentOp::createAndPopulate<ParallelOp>(ParallelOp, DeviceType,
482 OpBuilder &);
483template KernelEnvironmentOp
484KernelEnvironmentOp::createAndPopulate<KernelsOp>(KernelsOp, DeviceType,
485 OpBuilder &);
486template KernelEnvironmentOp
487KernelEnvironmentOp::createAndPopulate<SerialOp>(SerialOp, DeviceType,
488 OpBuilder &);
489
490LogicalResult KernelEnvironmentOp::verify() {
491 if (getAsyncOnly() && getAsyncOperand())
492 return emitError(message: "async-only cannot appear with async operand");
493 if (getWaitOnly() && (!getWaitOperands().empty() || getWaitDevnum()))
494 return emitError(message: "wait-only cannot appear with wait operands or devnum");
495 if ((*this)->getParentOfType<KernelEnvironmentOp>())
496 return emitError(
497 message: "must not be nested within another acc.kernel_environment");
498 return success();
499}
500
501//===----------------------------------------------------------------------===//
502// FirstprivateMapInitialOp
503//===----------------------------------------------------------------------===//
504
505LogicalResult FirstprivateMapInitialOp::verify() {
506 if (getDataClause() != acc::DataClause::acc_firstprivate)
507 return emitError(message: "data clause associated with firstprivate operation must "
508 "match its intent");
509 if (!getVar())
510 return emitError(message: "must have var operand");
511 if (!mlir::isa<mlir::acc::PointerLikeType>(Val: getVar().getType()) &&
512 !mlir::isa<mlir::acc::MappableType>(Val: getVar().getType()))
513 return emitError(message: "var must be mappable or pointer-like");
514 if (mlir::isa<mlir::acc::PointerLikeType>(Val: getVar().getType()) &&
515 getVarType() == getVar().getType())
516 return emitError(message: "varType must capture the element type of var");
517 if (getModifiers() != acc::DataClauseModifier::none)
518 return emitError(message: "no data clause modifiers are allowed");
519 return success();
520}
521
522void FirstprivateMapInitialOp::getEffects(
523 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
524 &effects) {
525 effects.emplace_back(Args: MemoryEffects::Read::get(),
526 Args: acc::CurrentDeviceIdResource::get());
527 addOperandEffect<MemoryEffects::Read>(effects, operand: getVarMutable());
528 addResultEffect<MemoryEffects::Write>(effects, result: getAccVar());
529}
530
531//===----------------------------------------------------------------------===//
532// ReductionInitOp
533//===----------------------------------------------------------------------===//
534
535void ReductionInitOp::getSuccessorRegions(
536 RegionBranchPoint point, SmallVectorImpl<RegionSuccessor> &regions) {
537 getSingleRegionOpSuccessorRegions(op: getOperation(), region&: getRegion(), point,
538 regions);
539}
540
541void ReductionInitOp::getRegionInvocationBounds(
542 ArrayRef<Attribute> operands,
543 SmallVectorImpl<InvocationBounds> &invocationBounds) {
544 invocationBounds.emplace_back(Args: 1, Args: 1);
545}
546
547ValueRange ReductionInitOp::getSuccessorInputs(RegionSuccessor successor) {
548 return getSingleRegionSuccessorInputs(op: getOperation(), successor);
549}
550
551LogicalResult ReductionInitOp::verify() {
552 Block &block = getRegion().front();
553 if (auto yieldOp = dyn_cast<acc::YieldOp>(Val: block.getTerminator())) {
554 if (yieldOp.getNumOperands() != 1)
555 return emitOpError(
556 message: "region must yield exactly one value (private storage)");
557 if (yieldOp.getOperand(i: 0).getType() != getVar().getType())
558 return emitOpError(message: "yielded value type must match var type");
559 }
560 return success();
561}
562
563//===----------------------------------------------------------------------===//
564// ReductionCombineRegionOp
565//===----------------------------------------------------------------------===//
566
567void ReductionCombineRegionOp::getSuccessorRegions(
568 RegionBranchPoint point, SmallVectorImpl<RegionSuccessor> &regions) {
569 getSingleRegionOpSuccessorRegions(op: getOperation(), region&: getRegion(), point,
570 regions);
571}
572
573void ReductionCombineRegionOp::getRegionInvocationBounds(
574 ArrayRef<Attribute> operands,
575 SmallVectorImpl<InvocationBounds> &invocationBounds) {
576 invocationBounds.emplace_back(Args: 1, Args: 1);
577}
578
579ValueRange
580ReductionCombineRegionOp::getSuccessorInputs(RegionSuccessor successor) {
581 return getSingleRegionSuccessorInputs(op: getOperation(), successor);
582}
583
584LogicalResult ReductionCombineRegionOp::verify() {
585 Block &block = getRegion().front();
586 if (auto yieldOp = dyn_cast<acc::YieldOp>(Val: block.getTerminator())) {
587 if (yieldOp.getNumOperands() != 0)
588 return emitOpError(message: "region must be terminated by acc.yield with no "
589 "operands");
590 }
591 return success();
592}
593
594//===----------------------------------------------------------------------===//
595// ReductionAccumulateOp
596//===----------------------------------------------------------------------===//
597
598LogicalResult ReductionAccumulateOp::verify() {
599 Type valueType = getValue().getType();
600 auto ptrLikeTy = cast<PointerLikeType>(Val: getMemref().getType());
601 Type elementType = ptrLikeTy.getElementType();
602 if (!elementType)
603 return emitOpError(message: "pointer-like destination must have an element type");
604 if (elementType != valueType)
605 return emitOpError(message: "pointer-like element type must match value type");
606 if (getParDims().getArray().empty())
607 return emitOpError(message: "par_dims must specify at least one parallel dimension");
608 return success();
609}
610
611//===----------------------------------------------------------------------===//
612// ReductionAccumulateArrayOp
613//===----------------------------------------------------------------------===//
614
615LogicalResult ReductionAccumulateArrayOp::verify() {
616 if (getParDims().getArray().empty())
617 return emitOpError(message: "par_dims must specify at least one parallel dimension");
618 return success();
619}
620
621//===----------------------------------------------------------------------===//
622// ReductionCombineOp
623//===----------------------------------------------------------------------===//
624
625void ReductionCombineOp::getEffects(
626 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
627 &effects) {
628 effects.emplace_back(Args: MemoryEffects::Read::get(), Args: &getSrcMemrefMutable(),
629 Args: SideEffects::DefaultResource::get());
630 effects.emplace_back(Args: MemoryEffects::Read::get(), Args: &getDestMemrefMutable(),
631 Args: SideEffects::DefaultResource::get());
632 effects.emplace_back(Args: MemoryEffects::Write::get(), Args: &getDestMemrefMutable(),
633 Args: SideEffects::DefaultResource::get());
634}
635
636//===----------------------------------------------------------------------===//
637// ComputeRegionOp
638//===----------------------------------------------------------------------===//
639
640static ParWidthOp getParWidthOpForLaunchArg(ComputeRegionOp op,
641 GPUParallelDimAttr parDim) {
642 for (auto launchArg : op.getLaunchArgs()) {
643 auto parOp = launchArg.getDefiningOp<ParWidthOp>();
644 if (!parOp)
645 continue;
646 auto launchArgDim = cast<GPUParallelDimAttr>(Val: parOp.getParDim());
647 if (launchArgDim == parDim)
648 return parOp;
649 }
650 return nullptr;
651}
652
653std::optional<Value> ComputeRegionOp::getLaunchArg(GPUParallelDimAttr parDim) {
654 if (auto parWidthOp = getParWidthOpForLaunchArg(op: *this, parDim))
655 return parWidthOp.getResult();
656 return {};
657}
658
659std::optional<Value>
660ComputeRegionOp::getKnownLaunchArg(GPUParallelDimAttr parDim) {
661 if (auto parWidthOp = getParWidthOpForLaunchArg(op: *this, parDim))
662 if (parWidthOp.getLaunchArg())
663 return parWidthOp.getLaunchArg();
664 return {};
665}
666
667std::optional<uint64_t>
668ComputeRegionOp::getKnownConstantLaunchArg(GPUParallelDimAttr parDim) {
669 auto knownParWidth = getKnownLaunchArg(parDim);
670 if (!knownParWidth.has_value())
671 return {};
672 std::optional<int64_t> width = getConstantIntValue(ofr: knownParWidth.value());
673 if (!width || *width < 0)
674 return {};
675 return static_cast<uint64_t>(*width);
676}
677
678BlockArgument ComputeRegionOp::appendInputArg(Value value) {
679 getInputArgsMutable().append(values: value);
680 return getBody()->addArgument(type: value.getType(), loc: getLoc());
681}
682
683std::optional<BlockArgument>
684ComputeRegionOp::wireHoistedValueThroughIns(Value value) {
685 Region &region = getRegion();
686
687 auto useIsInRegion = [&](OpOperand &use) -> bool {
688 return region.isAncestor(other: use.getOwner()->getParentRegion());
689 };
690
691 if (!areValuesDefinedAbove(values: ValueRange(value), limit&: region) ||
692 !llvm::any_of(Range: value.getUses(), P: useIsInRegion))
693 return std::nullopt;
694
695 BlockArgument arg = appendInputArg(value);
696 replaceAllUsesInRegionWith(orig: value, replacement: arg, region);
697 return arg;
698}
699
700bool ComputeRegionOp::isEffectivelySerial() {
701 auto *ctx = getContext();
702
703 if (getLaunchArg(parDim: GPUParallelDimAttr::seqDim(ctx)))
704 return true;
705
706 auto checkDim = [&](GPUParallelDimAttr dim) -> bool {
707 auto val = getKnownConstantLaunchArg(parDim: dim);
708 return val && *val == 1;
709 };
710
711 return checkDim(GPUParallelDimAttr::threadXDim(ctx)) &&
712 checkDim(GPUParallelDimAttr::threadYDim(ctx)) &&
713 checkDim(GPUParallelDimAttr::threadZDim(ctx)) &&
714 checkDim(GPUParallelDimAttr::blockXDim(ctx)) &&
715 checkDim(GPUParallelDimAttr::blockYDim(ctx)) &&
716 checkDim(GPUParallelDimAttr::blockZDim(ctx));
717}
718
719bool ComputeRegionOp::launchesMultipleBlocks() {
720 return llvm::any_of(Range: getLaunchParDims(), P: [&](GPUParallelDimAttr dim) {
721 if (!dim.isAnyBlock())
722 return false;
723 std::optional<uint64_t> width = getKnownConstantLaunchArg(parDim: dim);
724 return !width || *width != 1;
725 });
726}
727
728BlockArgument ComputeRegionOp::parDimToWidth(GPUParallelDimAttr parDim) {
729 for (auto [pos, launchArg] : llvm::enumerate(First: getLaunchArgs())) {
730 auto parOp = launchArg.getDefiningOp<ParWidthOp>();
731 assert(parOp);
732 auto launchArgDim = cast<GPUParallelDimAttr>(Val: parOp.getParDim());
733 if (launchArgDim == parDim) {
734 assert(pos < getRegion().front().getNumArguments() &&
735 "launch arg position out of range");
736 return getRegion().front().getArgument(i: pos);
737 }
738 }
739 llvm_unreachable("attempting to get unspecified parDim");
740}
741
742SmallVector<GPUParallelDimAttr> ComputeRegionOp::getLaunchParDims() {
743 SmallVector<GPUParallelDimAttr> parDims;
744 for (auto launchArg : getLaunchArgs()) {
745 auto parOp = launchArg.getDefiningOp<ParWidthOp>();
746 auto launchArgDim = cast<GPUParallelDimAttr>(Val: parOp.getParDim());
747 int64_t dimInt = launchArgDim.getValue().getInt();
748 parDims.push_back(Elt: intToParDim(context: getContext(), dimInt));
749 }
750 return parDims;
751}
752
753Value ComputeRegionOp::getOperand(BlockArgument blockArg) {
754 Block *body = getBody();
755 if (blockArg.getOwner() != body)
756 return Value();
757 unsigned argNumber = blockArg.getArgNumber();
758 unsigned numLaunchArgs = getLaunchArgs().size();
759 unsigned numInputArgs = getInputArgs().size();
760 if (argNumber >= numLaunchArgs + numInputArgs)
761 return Value();
762 if (argNumber < numLaunchArgs)
763 return getLaunchArgs()[argNumber];
764 return getInputArgs()[argNumber - numLaunchArgs];
765}
766
767std::optional<BlockArgument> ComputeRegionOp::getBlockArg(Value value) {
768 Block *body = getBody();
769 for (auto [idx, launchVal] : llvm::enumerate(First: getLaunchArgs())) {
770 if (launchVal == value)
771 return body->getArgument(i: idx);
772 }
773 unsigned numLaunch = getLaunchArgs().size();
774 for (auto [idx, inputVal] : llvm::enumerate(First: getInputArgs())) {
775 if (inputVal == value)
776 return body->getArgument(i: numLaunch + idx);
777 }
778 return std::nullopt;
779}
780
781void ComputeRegionOp::getCanonicalizationPatterns(RewritePatternSet &results,
782 MLIRContext *context) {
783 results.add<ComputeRegionRemoveDuplicateArgs, ComputeRegionRemoveUnusedArgs>(
784 arg&: context);
785}
786
787BlockArgument ComputeRegionOp::gpuParWidth(gpu::Processor processor) {
788 return parDimToWidth(parDim: GPUParallelDimAttr::get(getContext(), processor));
789}
790
791LogicalResult ComputeRegionOp::verify() {
792 for (auto op : getLaunchArgs())
793 if (!op.getDefiningOp<acc::ParWidthOp>())
794 return emitOpError(
795 message: "launch arguments must be results of acc.par_width operations");
796
797 unsigned expectedBlockArgs = getLaunchArgs().size() + getInputArgs().size();
798 unsigned actualBlockArgs = getRegion().front().getNumArguments();
799 if (expectedBlockArgs != actualBlockArgs)
800 return emitOpError(message: "expected ")
801 << expectedBlockArgs << " block arguments (launch + input), got "
802 << actualBlockArgs;
803
804 return success();
805}
806
807void ComputeRegionOp::print(OpAsmPrinter &p) {
808 ValueRange regionArgs = getBody()->getArguments();
809 ValueRange launchArgs = getLaunchArgs();
810 ValueRange inputArgs = getInputArgs();
811
812 assert(regionArgs.size() == (launchArgs.size() + inputArgs.size()) &&
813 "region args mismatch");
814
815 if (getStream())
816 p << " stream(" << getStream() << " : " << getStream().getType() << ")";
817
818 size_t i = 0;
819 if (!launchArgs.empty()) {
820 p << " launch(";
821 for (size_t j = 0; j < launchArgs.size(); ++j, ++i) {
822 p << regionArgs[i] << " = " << launchArgs[j];
823 if (j < launchArgs.size() - 1)
824 p << ", ";
825 }
826 p << ")";
827 }
828 if (!inputArgs.empty()) {
829 p << " ins(";
830 for (size_t j = 0; j < inputArgs.size(); ++j, ++i) {
831 p << regionArgs[i] << " = " << inputArgs[j];
832 if (j < inputArgs.size() - 1)
833 p << ", ";
834 }
835 p << ") : (";
836 for (size_t j = 0; j < inputArgs.size(); ++j) {
837 p << inputArgs[j].getType();
838 if (j < inputArgs.size() - 1)
839 p << ", ";
840 }
841 p << ")";
842 }
843 p.printOptionalArrowTypeList(types: getResultTypes());
844 p << " ";
845 p.printRegion(blocks&: getRegion(), /*printEntryBlockArgs=*/false);
846 ComputeRegionOp::printProperties(ctx: getContext(), p, properties: getProperties(),
847 /*elidedProps=*/getOperandSegmentSizeAttr());
848 p.printOptionalAttrDict(attrs: (*this)->getDiscardableAttrDictionary().getValue());
849}
850
851ParseResult ComputeRegionOp::parse(OpAsmParser &parser,
852 OperationState &result) {
853 auto &builder = parser.getBuilder();
854
855 SmallVector<OpAsmParser::Argument> regionArgs;
856 OpAsmParser::UnresolvedOperand streamOperand;
857 Type streamType;
858 SmallVector<OpAsmParser::UnresolvedOperand> launchOperands;
859 SmallVector<OpAsmParser::UnresolvedOperand> inputOperands;
860 SmallVector<Type> types;
861
862 bool hasStream = false;
863 if (succeeded(Result: parser.parseOptionalKeyword(keyword: "stream"))) {
864 hasStream = true;
865 if (parser.parseLParen() || parser.parseOperand(result&: streamOperand) ||
866 parser.parseColon() || parser.parseType(result&: streamType) ||
867 parser.parseRParen())
868 return failure();
869 }
870
871 if (succeeded(Result: parser.parseOptionalKeyword(keyword: "launch"))) {
872 if (parser.parseAssignmentList(lhs&: regionArgs, rhs&: launchOperands))
873 return failure();
874 Type indexType = builder.getIndexType();
875 for (size_t i = 0; i < regionArgs.size(); ++i)
876 types.push_back(Elt: indexType);
877 }
878
879 if (succeeded(Result: parser.parseOptionalKeyword(keyword: "ins"))) {
880 if (parser.parseAssignmentList(lhs&: regionArgs, rhs&: inputOperands) ||
881 parser.parseColon() || parser.parseLParen() ||
882 parser.parseTypeList(result&: types) || parser.parseRParen())
883 return failure();
884 }
885
886 if (parser.parseOptionalArrowTypeList(result&: result.types))
887 return failure();
888
889 for (auto [iterArg, type] : llvm::zip_equal(t&: regionArgs, u&: types))
890 iterArg.type = type;
891
892 Region *body = result.addRegion();
893 if (parser.parseRegion(region&: *body, arguments: regionArgs))
894 return failure();
895 ComputeRegionOp::ensureTerminator(region&: *body, builder&: parser.getBuilder(),
896 loc: result.location);
897
898 const size_t numLaunchOperands = launchOperands.size();
899 const size_t numInputOperands = inputOperands.size();
900 assert(numLaunchOperands + numInputOperands == regionArgs.size() &&
901 "compute region args mismatch");
902
903 DenseI32ArrayAttr operandSegmentSizes = builder.getDenseI32ArrayAttr(
904 values: {static_cast<int32_t>(numLaunchOperands),
905 static_cast<int32_t>(numInputOperands), hasStream ? 1 : 0});
906
907 for (size_t i = 0; i < numLaunchOperands; ++i) {
908 if (parser.resolveOperand(operand: launchOperands[i], type: types[i], result&: result.operands))
909 return failure();
910 }
911
912 for (size_t i = numLaunchOperands; i < regionArgs.size(); ++i) {
913 if (parser.resolveOperand(operand: inputOperands[i - numLaunchOperands], type: types[i],
914 result&: result.operands))
915 return failure();
916 }
917
918 if (hasStream) {
919 if (parser.resolveOperand(operand: streamOperand, type: streamType, result&: result.operands))
920 return failure();
921 }
922
923 Attribute parsedProperties;
924 if (ComputeRegionOp::genericParseProperties(parser, result&: parsedProperties))
925 return failure();
926 auto propertyDictionary = dyn_cast_or_null<DictionaryAttr>(Val&: parsedProperties);
927 if (parsedProperties && !propertyDictionary)
928 return parser.emitError(loc: parser.getNameLoc(),
929 message: "expected properties dictionary");
930
931 NamedAttrList properties(propertyDictionary ? propertyDictionary
932 : builder.getDictionaryAttr(value: {}));
933 properties.set(name: ComputeRegionOp::getOperandSegmentSizeAttr(),
934 value: operandSegmentSizes);
935 propertyDictionary = properties.getDictionary(context: builder.getContext());
936 auto emitError = [&]() {
937 return mlir::emitError(loc: result.location, message: "invalid properties ")
938 << propertyDictionary << " for op " << result.name.getStringRef()
939 << ": ";
940 };
941 if (failed(Result: ComputeRegionOp::setPropertiesFromParsedAttr(
942 prop&: result.getOrAddProperties<Properties>(), attr: propertyDictionary,
943 emitError)))
944 return failure();
945
946 auto attrsLoc = parser.getCurrentLocation();
947 if (parser.parseOptionalAttrDict(result&: result.attributes))
948 return failure();
949 for (StringRef attrName : ComputeRegionOp::getAttributeNames()) {
950 if (result.attributes.get(name: attrName))
951 return parser.emitError(loc: attrsLoc)
952 << "inherent attribute '" << attrName
953 << "' cannot be parsed from attr-dict when strict properties in "
954 "assembly format is enabled";
955 }
956
957 return success();
958}
959
960//===----------------------------------------------------------------------===//
961// GPUSharedMemoryOp
962//===----------------------------------------------------------------------===//
963
964LogicalResult GPUSharedMemoryOp::verify() {
965 if (getNumCopies() <= 0)
966 return emitOpError(message: "num_copies must be positive");
967 if (getStaticUpperBoundBytes() <= 0)
968 return emitOpError(message: "static_upper_bound_bytes must be positive");
969
970 bool hasScaling = static_cast<bool>(getDynamicSharedMemoryScalingBytes());
971 bool hasFixed = static_cast<bool>(getDynamicSharedMemoryFixedBytes());
972 if (hasScaling != hasFixed)
973 return emitOpError(
974 message: "dynamic_shared_memory_scaling_bytes and "
975 "dynamic_shared_memory_fixed_bytes must both be present or both be "
976 "absent");
977 if (auto scalingAttr = getDynamicSharedMemoryScalingBytesAttr())
978 if (scalingAttr.getValue().isNegative())
979 return emitOpError(message: "dynamic_shared_memory_scaling_bytes must be "
980 "non-negative");
981 if (auto fixedAttr = getDynamicSharedMemoryFixedBytesAttr())
982 if (fixedAttr.getValue().isNegative())
983 return emitOpError(message: "dynamic_shared_memory_fixed_bytes must be "
984 "non-negative");
985
986 auto resultTy = cast<MemRefType>(Val: getResult().getType());
987 auto addrSpace =
988 dyn_cast_if_present<gpu::AddressSpaceAttr>(Val: resultTy.getMemorySpace());
989 if (!addrSpace || addrSpace.getValue() != gpu::AddressSpace::Workgroup)
990 return emitOpError(message: "result memref must use #gpu.address_space<workgroup>");
991
992 return success();
993}
994
995//===----------------------------------------------------------------------===//
996// PredicateRegionOp
997//===----------------------------------------------------------------------===//
998
999LogicalResult PredicateRegionOp::verify() {
1000 if (getRegion().empty())
1001 return emitOpError(message: "region needs to have at least one block");
1002 if (getRegion().front().getNumArguments() > 0)
1003 return emitOpError(message: "region cannot have any arguments");
1004 if (!getOperation()->getParentOfType<ComputeRegionOp>())
1005 return emitOpError(message: "must be nested within an acc.compute_region operation");
1006 return success();
1007}
1008
1009//===----------------------------------------------------------------------===//
1010// MapInfoOp
1011//===----------------------------------------------------------------------===//
1012
1013LogicalResult MapInfoOp::verify() {
1014 // A pointer-like var addresses the mapped object, so varType has to name that
1015 // object rather than the address of it.
1016 if (mlir::isa<acc::PointerLikeType>(Val: getVar().getType()) &&
1017 getVarType() == getVar().getType())
1018 return emitOpError(message: "varType must capture the element type of var");
1019
1020 // A descriptor operand is only meaningful together with the layout it
1021 // follows. Without a kind, consumers have no way to interpret it.
1022 if (getDesc() && getDescKind() == DataDescKind::none)
1023 return emitOpError(message: "desc requires a descKind other than none");
1024
1025 // Bounds count elements of the mapped object, which is the OpenACC
1026 // descriptor layout, so it must be among the kinds named here.
1027 if (!getBounds().empty() &&
1028 !acc::bitEnumContainsAny(bits: getDescKind(), bit: DataDescKind::openacc))
1029 return emitOpError(message: "bounds require descKind openacc");
1030
1031 // Anything below -1 has no meaning: -1 states that the size is unknown at
1032 // compile time and 0 defers it to bounds or to a descriptor.
1033 if (getSize()) {
1034 std::optional<int64_t> constantSize = getConstantIntValue(ofr: getSize());
1035 if (constantSize && *constantSize < -1)
1036 return emitOpError(message: "size must be -1, 0, or a positive byte count");
1037 }
1038
1039 return success();
1040}
1041
1042//===----------------------------------------------------------------------===//
1043// GPUParallelDimAttr
1044//===----------------------------------------------------------------------===//
1045
1046GPUParallelDimAttr GPUParallelDimAttr::get(MLIRContext *context,
1047 gpu::Processor proc) {
1048 return processorParDim(context, proc);
1049}
1050
1051GPUParallelDimAttr GPUParallelDimAttr::seqDim(MLIRContext *context) {
1052 return processorParDim(context, proc: gpu::Processor::Sequential);
1053}
1054
1055GPUParallelDimAttr GPUParallelDimAttr::threadXDim(MLIRContext *context) {
1056 return processorParDim(context, proc: gpu::Processor::ThreadX);
1057}
1058
1059GPUParallelDimAttr GPUParallelDimAttr::threadYDim(MLIRContext *context) {
1060 return processorParDim(context, proc: gpu::Processor::ThreadY);
1061}
1062
1063GPUParallelDimAttr GPUParallelDimAttr::threadZDim(MLIRContext *context) {
1064 return processorParDim(context, proc: gpu::Processor::ThreadZ);
1065}
1066
1067GPUParallelDimAttr GPUParallelDimAttr::blockXDim(MLIRContext *context) {
1068 return processorParDim(context, proc: gpu::Processor::BlockX);
1069}
1070
1071GPUParallelDimAttr GPUParallelDimAttr::blockYDim(MLIRContext *context) {
1072 return processorParDim(context, proc: gpu::Processor::BlockY);
1073}
1074
1075GPUParallelDimAttr GPUParallelDimAttr::blockZDim(MLIRContext *context) {
1076 return processorParDim(context, proc: gpu::Processor::BlockZ);
1077}
1078
1079Attribute GPUParallelDimAttr::parse(AsmParser &parser, Type type) {
1080 GPUParallelDimAttr dim;
1081 if (parser.parseLess() || parseProcessorValue(parser, dim) ||
1082 parser.parseGreater()) {
1083 parser.emitError(loc: parser.getCurrentLocation(),
1084 message: "expected format `<` processor_name `>`");
1085 return {};
1086 }
1087 return dim;
1088}
1089
1090void GPUParallelDimAttr::print(AsmPrinter &printer) const {
1091 printer << "<";
1092 printProcessorValue(printer, attr: *this);
1093 printer << ">";
1094}
1095
1096GPUParallelDimAttr GPUParallelDimAttr::threadDim(MLIRContext *context,
1097 unsigned index) {
1098 assert(index <= 2 && "thread dimension index must be 0, 1, or 2");
1099 switch (index) {
1100 case 0:
1101 return threadXDim(context);
1102 case 1:
1103 return threadYDim(context);
1104 case 2:
1105 return threadZDim(context);
1106 }
1107 llvm_unreachable("validated thread dimension index");
1108}
1109
1110GPUParallelDimAttr GPUParallelDimAttr::blockDim(MLIRContext *context,
1111 unsigned index) {
1112 assert(index <= 2 && "block dimension index must be 0, 1, or 2");
1113 switch (index) {
1114 case 0:
1115 return blockXDim(context);
1116 case 1:
1117 return blockYDim(context);
1118 case 2:
1119 return blockZDim(context);
1120 }
1121 llvm_unreachable("validated block dimension index");
1122}
1123
1124gpu::Processor GPUParallelDimAttr::getProcessor() const {
1125 return indexToGpuProcessor(idx: getValue().getInt());
1126}
1127
1128int GPUParallelDimAttr::getOrder() const {
1129 return gpuProcessorIndex(p: getProcessor());
1130}
1131
1132GPUParallelDimAttr GPUParallelDimAttr::getOneHigher() const {
1133 int order = getOrder();
1134 if (order >= 6) // BlockZ is the highest
1135 return *this;
1136 return get(context: getContext(), proc: indexToGpuProcessor(idx: order + 1));
1137}
1138
1139GPUParallelDimAttr GPUParallelDimAttr::getOneLower() const {
1140 int order = getOrder();
1141 if (order <= 0) // Sequential is the lowest
1142 return *this;
1143 return get(context: getContext(), proc: indexToGpuProcessor(idx: order - 1));
1144}
1145
1146bool GPUParallelDimAttr::isSeq() const {
1147 return getProcessor() == gpu::Processor::Sequential;
1148}
1149bool GPUParallelDimAttr::isThreadX() const {
1150 return getProcessor() == gpu::Processor::ThreadX;
1151}
1152bool GPUParallelDimAttr::isThreadY() const {
1153 return getProcessor() == gpu::Processor::ThreadY;
1154}
1155bool GPUParallelDimAttr::isThreadZ() const {
1156 return getProcessor() == gpu::Processor::ThreadZ;
1157}
1158bool GPUParallelDimAttr::isBlockX() const {
1159 return getProcessor() == gpu::Processor::BlockX;
1160}
1161bool GPUParallelDimAttr::isBlockY() const {
1162 return getProcessor() == gpu::Processor::BlockY;
1163}
1164bool GPUParallelDimAttr::isBlockZ() const {
1165 return getProcessor() == gpu::Processor::BlockZ;
1166}
1167bool GPUParallelDimAttr::isAnyThread() const {
1168 return isThreadX() || isThreadY() || isThreadZ();
1169}
1170bool GPUParallelDimAttr::isAnyBlock() const {
1171 return isBlockX() || isBlockY() || isBlockZ();
1172}
1173
1174//===----------------------------------------------------------------------===//
1175// GPUParallelDimsAttr
1176//===----------------------------------------------------------------------===//
1177
1178GPUParallelDimsAttr GPUParallelDimsAttr::seq(MLIRContext *ctx) {
1179 return GPUParallelDimsAttr::get(context: ctx, array: {GPUParallelDimAttr::seqDim(context: ctx)});
1180}
1181
1182bool GPUParallelDimsAttr::isSeq() const {
1183 assert(!getArray().empty() && "no par_dims found");
1184 if (getArray().size() == 1) {
1185 auto parDim = dyn_cast<GPUParallelDimAttr>(Val: getArray()[0]);
1186 assert(parDim && "expected GPUParallelDimAttr");
1187 return parDim.isSeq();
1188 }
1189 return false;
1190}
1191
1192bool GPUParallelDimsAttr::isParallel() const { return !isSeq(); }
1193
1194bool GPUParallelDimsAttr::isMultiDim() const { return getArray().size() > 1; }
1195
1196bool GPUParallelDimsAttr::hasAnyBlockLevel() const {
1197 return llvm::any_of(
1198 Range: getArray(), P: [](const GPUParallelDimAttr &p) { return p.isAnyBlock(); });
1199}
1200
1201bool GPUParallelDimsAttr::hasOnlyBlockLevel() const {
1202 return !getArray().empty() &&
1203 llvm::all_of(Range: getArray(), P: [](const GPUParallelDimAttr &p) {
1204 return p.isAnyBlock();
1205 });
1206}
1207
1208bool GPUParallelDimsAttr::hasOnlyThreadYLevel() const {
1209 return !getArray().empty() &&
1210 llvm::all_of(Range: getArray(), P: [](const GPUParallelDimAttr &p) {
1211 return p.isThreadY();
1212 });
1213}
1214
1215bool GPUParallelDimsAttr::hasOnlyThreadXLevel() const {
1216 return !getArray().empty() &&
1217 llvm::all_of(Range: getArray(), P: [](const GPUParallelDimAttr &p) {
1218 return p.isThreadX();
1219 });
1220}
1221
1222Attribute GPUParallelDimsAttr::parse(AsmParser &parser, Type type) {
1223 FailureOr<SmallVector<GPUParallelDimAttr>> parDims =
1224 parseGPUParallelDimList(parser);
1225 if (failed(Result: parDims))
1226 return {};
1227 return GPUParallelDimsAttr::get(context: parser.getContext(), array: *parDims);
1228}
1229
1230void GPUParallelDimsAttr::print(AsmPrinter &printer) const {
1231 printGPUParallelDimList(printer, dims: getArray());
1232}
1233
1234//===----------------------------------------------------------------------===//
1235// ActiveParDimsAttr
1236//===----------------------------------------------------------------------===//
1237
1238Attribute ActiveParDimsAttr::parse(AsmParser &parser, Type type) {
1239 FailureOr<SmallVector<GPUParallelDimAttr>> parDims =
1240 parseGPUParallelDimList(parser);
1241 if (failed(Result: parDims))
1242 return {};
1243 return ActiveParDimsAttr::get(context: parser.getContext(), array: *parDims);
1244}
1245
1246void ActiveParDimsAttr::print(AsmPrinter &printer) const {
1247 printGPUParallelDimList(printer, dims: getArray());
1248}
1249