1//===- OpenACC.cpp - OpenACC MLIR Operations ------------------------------===//
2//
3// Part of the MLIR Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Dialect/OpenACC/OpenACC.h"
10#include "mlir/Dialect/Arith/IR/Arith.h"
11#include "mlir/Dialect/Utils/StaticValueUtils.h"
12#include "mlir/IR/Builders.h"
13#include "mlir/IR/BuiltinAttributes.h"
14#include "mlir/IR/BuiltinOps.h"
15#include "mlir/IR/BuiltinTypes.h"
16#include "mlir/IR/IRMapping.h"
17#include "mlir/IR/Matchers.h"
18#include "mlir/IR/OpImplementation.h"
19#include "mlir/IR/SymbolTable.h"
20#include "mlir/Support/LLVM.h"
21#include "mlir/Transforms/DialectConversion.h"
22#include "llvm/ADT/SmallSet.h"
23#include "llvm/ADT/TypeSwitch.h"
24#include "llvm/Support/LogicalResult.h"
25#include <variant>
26
27using namespace mlir;
28using namespace acc;
29
30#include "mlir/Dialect/OpenACC/OpenACCOpsDialect.cpp.inc"
31#include "mlir/Dialect/OpenACC/OpenACCOpsEnums.cpp.inc"
32#include "mlir/Dialect/OpenACC/OpenACCOpsInterfaces.cpp.inc"
33#include "mlir/Dialect/OpenACC/OpenACCTypeInterfaces.cpp.inc"
34#include "mlir/Dialect/OpenACCMPCommon/Interfaces/OpenACCMPOpsInterfaces.cpp.inc"
35
36namespace {
37struct PrivateTypePointerLikeModel
38 : public PointerLikeType::ExternalModel<PrivateTypePointerLikeModel,
39 PrivateType> {
40 Type getElementType(Type type) const {
41 return cast<PrivateType>(Val&: type).getBaseTy();
42 }
43
44 Value genCast(Type, OpBuilder &builder, Location loc, Value value,
45 Type resultType) const {
46 if (value.getType() == resultType)
47 return value;
48 if (!isa<PointerLikeType>(Val: resultType))
49 return {};
50 return UnwrapPrivateOp::create(builder, location: loc, result: resultType, handle: value).getResult();
51 }
52
53 MemRefType getAsMemRefType(Type type, ModuleOp module) const {
54 Type baseTy = cast<PrivateType>(Val&: type).getBaseTy();
55 if (auto memrefTy = dyn_cast<MemRefType>(Val&: baseTy))
56 return memrefTy;
57 if (auto ptrLikeTy = dyn_cast<PointerLikeType>(Val&: baseTy))
58 return ptrLikeTy.getAsMemRefType(module);
59 return {};
60 }
61};
62
63/// Helper function for any of the times we need to modify an ArrayAttr based on
64/// a device type list. Returns a new ArrayAttr with all of the
65/// existingDeviceTypes, plus the effective new ones(or an added none if hte new
66/// list is empty).
67mlir::ArrayAttr addDeviceTypeAffectedOperandHelper(
68 MLIRContext *context, mlir::ArrayAttr existingDeviceTypes,
69 llvm::ArrayRef<acc::DeviceType> newDeviceTypes) {
70 llvm::SmallVector<mlir::Attribute> deviceTypes;
71 if (existingDeviceTypes)
72 llvm::copy(Range&: existingDeviceTypes, Out: std::back_inserter(x&: deviceTypes));
73
74 if (newDeviceTypes.empty())
75 deviceTypes.push_back(
76 Elt: acc::DeviceTypeAttr::get(context, value: acc::DeviceType::None));
77
78 for (DeviceType dt : newDeviceTypes)
79 deviceTypes.push_back(Elt: acc::DeviceTypeAttr::get(context, value: dt));
80
81 return mlir::ArrayAttr::get(context, value: deviceTypes);
82}
83
84/// Helper function for any of the times we need to add operands that are
85/// affected by a device type list. Returns a new ArrayAttr with all of the
86/// existingDeviceTypes, plus the effective new ones (or an added none, if the
87/// new list is empty). Additionally, adds the arguments to the argCollection
88/// the correct number of times. This will also update a 'segments' array, even
89/// if it won't be used.
90mlir::ArrayAttr addDeviceTypeAffectedOperandHelper(
91 MLIRContext *context, mlir::ArrayAttr existingDeviceTypes,
92 llvm::ArrayRef<acc::DeviceType> newDeviceTypes, mlir::ValueRange arguments,
93 mlir::MutableOperandRange argCollection,
94 llvm::SmallVector<int32_t> &segments) {
95 llvm::SmallVector<mlir::Attribute> deviceTypes;
96 if (existingDeviceTypes)
97 llvm::copy(Range&: existingDeviceTypes, Out: std::back_inserter(x&: deviceTypes));
98
99 if (newDeviceTypes.empty()) {
100 argCollection.append(values: arguments);
101 segments.push_back(Elt: arguments.size());
102 deviceTypes.push_back(
103 Elt: acc::DeviceTypeAttr::get(context, value: acc::DeviceType::None));
104 }
105
106 for (DeviceType dt : newDeviceTypes) {
107 argCollection.append(values: arguments);
108 segments.push_back(Elt: arguments.size());
109 deviceTypes.push_back(Elt: acc::DeviceTypeAttr::get(context, value: dt));
110 }
111
112 return mlir::ArrayAttr::get(context, value: deviceTypes);
113}
114
115/// Overload for when the 'segments' aren't needed.
116mlir::ArrayAttr addDeviceTypeAffectedOperandHelper(
117 MLIRContext *context, mlir::ArrayAttr existingDeviceTypes,
118 llvm::ArrayRef<acc::DeviceType> newDeviceTypes, mlir::ValueRange arguments,
119 mlir::MutableOperandRange argCollection) {
120 llvm::SmallVector<int32_t> segments;
121 return addDeviceTypeAffectedOperandHelper(context, existingDeviceTypes,
122 newDeviceTypes, arguments,
123 argCollection, segments);
124}
125} // namespace
126
127//===----------------------------------------------------------------------===//
128// OpenACC operations
129//===----------------------------------------------------------------------===//
130
131void OpenACCDialect::initialize() {
132 addOperations<
133#define GET_OP_LIST
134#include "mlir/Dialect/OpenACC/OpenACCOps.cpp.inc"
135 >();
136 addAttributes<
137#define GET_ATTRDEF_LIST
138#include "mlir/Dialect/OpenACC/OpenACCOpsAttributes.cpp.inc"
139 >();
140 addTypes<
141#define GET_TYPEDEF_LIST
142#include "mlir/Dialect/OpenACC/OpenACCOpsTypes.cpp.inc"
143 >();
144
145 PrivateType::attachInterface<PrivateTypePointerLikeModel>(context&: *getContext());
146}
147
148//===----------------------------------------------------------------------===//
149// RegionBranchOpInterface for acc.kernels / acc.parallel / acc.serial /
150// acc.kernel_environment / acc.data / acc.host_data / acc.loop
151//===----------------------------------------------------------------------===//
152
153/// Generic helper for single-region OpenACC ops that execute their body once
154/// and then continue after the operation with their results (if any).
155static void
156getSingleRegionOpSuccessorRegions(Operation *op, Region &region,
157 RegionBranchPoint point,
158 SmallVectorImpl<RegionSuccessor> &regions) {
159 if (point.isParent()) {
160 regions.push_back(Elt: RegionSuccessor(&region));
161 return;
162 }
163
164 regions.push_back(Elt: RegionSuccessor(op));
165}
166
167static ValueRange getSingleRegionSuccessorInputs(Operation *op,
168 RegionSuccessor successor) {
169 return successor.isOperation() ? ValueRange(op->getResults()) : ValueRange();
170}
171
172void KernelsOp::getSuccessorRegions(RegionBranchPoint point,
173 SmallVectorImpl<RegionSuccessor> &regions) {
174 getSingleRegionOpSuccessorRegions(op: getOperation(), region&: getRegion(), point,
175 regions);
176}
177
178ValueRange KernelsOp::getSuccessorInputs(RegionSuccessor successor) {
179 return getSingleRegionSuccessorInputs(op: getOperation(), successor);
180}
181
182void ParallelOp::getSuccessorRegions(
183 RegionBranchPoint point, SmallVectorImpl<RegionSuccessor> &regions) {
184 getSingleRegionOpSuccessorRegions(op: getOperation(), region&: getRegion(), point,
185 regions);
186}
187
188ValueRange ParallelOp::getSuccessorInputs(RegionSuccessor successor) {
189 return getSingleRegionSuccessorInputs(op: getOperation(), successor);
190}
191
192void SerialOp::getSuccessorRegions(RegionBranchPoint point,
193 SmallVectorImpl<RegionSuccessor> &regions) {
194 getSingleRegionOpSuccessorRegions(op: getOperation(), region&: getRegion(), point,
195 regions);
196}
197
198ValueRange SerialOp::getSuccessorInputs(RegionSuccessor successor) {
199 return getSingleRegionSuccessorInputs(op: getOperation(), successor);
200}
201
202void DataOp::getSuccessorRegions(RegionBranchPoint point,
203 SmallVectorImpl<RegionSuccessor> &regions) {
204 getSingleRegionOpSuccessorRegions(op: getOperation(), region&: getRegion(), point,
205 regions);
206}
207
208ValueRange DataOp::getSuccessorInputs(RegionSuccessor successor) {
209 return getSingleRegionSuccessorInputs(op: getOperation(), successor);
210}
211
212void HostDataOp::getSuccessorRegions(
213 RegionBranchPoint point, SmallVectorImpl<RegionSuccessor> &regions) {
214 getSingleRegionOpSuccessorRegions(op: getOperation(), region&: getRegion(), point,
215 regions);
216}
217
218ValueRange HostDataOp::getSuccessorInputs(RegionSuccessor successor) {
219 return getSingleRegionSuccessorInputs(op: getOperation(), successor);
220}
221
222/// Whether the body of a structured `acc.loop` is proven to run. This decides
223/// which edges out of the parent are feasible; the edges out of the region are
224/// unaffected.
225enum class BodyExecution {
226 /// The body runs at least once, so the parent cannot bypass the region.
227 Always,
228 /// The body never runs, so the parent cannot enter the region.
229 Never,
230 /// Neither could be proven, so the parent may do either.
231 Maybe
232};
233
234/// Prove whether the body of `loopOp` runs. A counted dimension whose entry
235/// test already fails at its lower bound runs exactly zero times, so a single
236/// comparison decides both `Always` and `Never`. Bounds that are not constant
237/// prove nothing.
238static BodyExecution getBodyExecution(LoopOp loopOp) {
239 // A container-like loop describes its iteration space inside the region.
240 if (loopOp.isContainerLike())
241 return BodyExecution::Maybe;
242
243 // The verifier guarantees one lower bound, upper bound and step per
244 // dimension.
245 ValueRange lbs = loopOp.getLowerbound();
246 ValueRange ubs = loopOp.getUpperbound();
247 ValueRange steps = loopOp.getStep();
248
249 BodyExecution result = BodyExecution::Always;
250 for (unsigned i = 0, e = lbs.size(); i < e; ++i) {
251 std::optional<int64_t> lb = getConstantIntValue(ofr: lbs[i]);
252 std::optional<int64_t> ub = getConstantIntValue(ofr: ubs[i]);
253 std::optional<int64_t> step = getConstantIntValue(ofr: steps[i]);
254 // An unknown bound cannot be tested, and a zero step either spins forever
255 // or never starts. Neither proves `Always`, but a later dimension may
256 // still prove the nest empty: `(0 to %n)` collapsed with `(0 to 0)`.
257 if (!lb || !ub || !step || *step == 0) {
258 result = BodyExecution::Maybe;
259 continue;
260 }
261
262 // The entry test at the lower bound. A descending dimension compares
263 // against its bound the other way round. The attribute is absent when
264 // every dimension is exclusive as in `scf.for`, and the verifier otherwise
265 // guarantees one entry per dimension.
266 std::optional<ArrayRef<bool>> inclusiveUbs =
267 loopOp.getInclusiveUpperbound();
268 bool inclusiveUb = inclusiveUbs && (*inclusiveUbs)[i];
269 assert(*step != 0 && "zero step should have been filtered out");
270 bool runsOnce = *step > 0 ? (inclusiveUb ? *lb <= *ub : *lb < *ub)
271 : (inclusiveUb ? *lb >= *ub : *lb > *ub);
272 // The dimensions are iterated as a nest, so one empty dimension empties
273 // the whole nest whatever the others do, while the body runs only if every
274 // dimension runs.
275 if (!runsOnce)
276 return BodyExecution::Never;
277 }
278
279 // No dimension was empty, so the body runs unless some dimension was
280 // unknown.
281 return result;
282}
283
284void LoopOp::getSuccessorRegions(RegionBranchPoint point,
285 SmallVectorImpl<RegionSuccessor> &regions) {
286 // Unstructured loops: the body may contain arbitrary CFG and early exits.
287 // At the RegionBranch level, only model entry into the body and exit to the
288 // parent; any backedges are represented inside the region CFG.
289 if (getUnstructured()) {
290 if (point.isParent()) {
291 regions.push_back(Elt: RegionSuccessor(&getRegion()));
292 return;
293 }
294 regions.push_back(Elt: RegionSuccessor(getOperation()));
295 return;
296 }
297
298 // Structured loops: model a loop-shaped region graph similar to scf.for,
299 // minus the entry edge the loop is proven not to take.
300 if (point.isParent()) {
301 switch (getBodyExecution(loopOp: *this)) {
302 case BodyExecution::Always:
303 regions.push_back(Elt: RegionSuccessor(&getRegion()));
304 return;
305 case BodyExecution::Never:
306 regions.push_back(Elt: RegionSuccessor(getOperation()));
307 return;
308 case BodyExecution::Maybe:
309 break;
310 }
311 }
312
313 regions.push_back(Elt: RegionSuccessor(&getRegion()));
314 regions.push_back(Elt: RegionSuccessor(getOperation()));
315}
316
317ValueRange LoopOp::getSuccessorInputs(RegionSuccessor successor) {
318 return getSingleRegionSuccessorInputs(op: getOperation(), successor);
319}
320
321//===----------------------------------------------------------------------===//
322// RegionBranchTerminatorOpInterface
323//===----------------------------------------------------------------------===//
324
325MutableOperandRange
326TerminatorOp::getMutableSuccessorOperands(RegionSuccessor /*point*/) {
327 // `acc.terminator` does not forward operands.
328 return MutableOperandRange(getOperation(), /*start=*/0, /*length=*/0);
329}
330
331//===----------------------------------------------------------------------===//
332// device_type support helpers
333//===----------------------------------------------------------------------===//
334
335static bool hasDeviceTypeValues(std::optional<mlir::ArrayAttr> arrayAttr) {
336 return arrayAttr && *arrayAttr && arrayAttr->size() > 0;
337}
338
339static bool hasDeviceType(std::optional<mlir::ArrayAttr> arrayAttr,
340 mlir::acc::DeviceType deviceType) {
341 if (!hasDeviceTypeValues(arrayAttr))
342 return false;
343
344 for (auto attr : *arrayAttr) {
345 auto deviceTypeAttr = mlir::dyn_cast<mlir::acc::DeviceTypeAttr>(Val&: attr);
346 if (deviceTypeAttr.getValue() == deviceType)
347 return true;
348 }
349
350 return false;
351}
352
353static void printDeviceTypes(mlir::OpAsmPrinter &p,
354 std::optional<mlir::ArrayAttr> deviceTypes) {
355 if (!hasDeviceTypeValues(arrayAttr: deviceTypes))
356 return;
357
358 p << "[";
359 llvm::interleaveComma(c: *deviceTypes, os&: p,
360 each_fn: [&](mlir::Attribute attr) { p << attr; });
361 p << "]";
362}
363
364static std::optional<unsigned> findSegment(ArrayAttr segments,
365 mlir::acc::DeviceType deviceType) {
366 unsigned segmentIdx = 0;
367 for (auto attr : segments) {
368 auto deviceTypeAttr = mlir::dyn_cast<mlir::acc::DeviceTypeAttr>(Val&: attr);
369 if (deviceTypeAttr.getValue() == deviceType)
370 return std::make_optional(t&: segmentIdx);
371 ++segmentIdx;
372 }
373 return std::nullopt;
374}
375
376static mlir::Operation::operand_range
377getValuesFromSegments(std::optional<mlir::ArrayAttr> arrayAttr,
378 mlir::Operation::operand_range range,
379 std::optional<llvm::ArrayRef<int32_t>> segments,
380 mlir::acc::DeviceType deviceType) {
381 if (!arrayAttr)
382 return range.take_front(n: 0);
383 if (auto pos = findSegment(segments: *arrayAttr, deviceType)) {
384 int32_t nbOperandsBefore = 0;
385 for (unsigned i = 0; i < *pos; ++i)
386 nbOperandsBefore += (*segments)[i];
387 return range.drop_front(n: nbOperandsBefore).take_front(n: (*segments)[*pos]);
388 }
389 return range.take_front(n: 0);
390}
391
392static mlir::Value
393getWaitDevnumValue(std::optional<mlir::ArrayAttr> deviceTypeAttr,
394 mlir::Operation::operand_range operands,
395 std::optional<llvm::ArrayRef<int32_t>> segments,
396 std::optional<mlir::ArrayAttr> hasWaitDevnum,
397 mlir::acc::DeviceType deviceType) {
398 if (!hasDeviceTypeValues(arrayAttr: deviceTypeAttr))
399 return {};
400 if (auto pos = findSegment(segments: *deviceTypeAttr, deviceType)) {
401 if (hasWaitDevnum && *hasWaitDevnum) {
402 auto boolAttr = mlir::dyn_cast<mlir::BoolAttr>(Val: (*hasWaitDevnum)[*pos]);
403 if (boolAttr && boolAttr.getValue())
404 return getValuesFromSegments(arrayAttr: deviceTypeAttr, range: operands, segments,
405 deviceType)
406 .front();
407 }
408 }
409 return {};
410}
411
412static mlir::Operation::operand_range
413getWaitValuesWithoutDevnum(std::optional<mlir::ArrayAttr> deviceTypeAttr,
414 mlir::Operation::operand_range operands,
415 std::optional<llvm::ArrayRef<int32_t>> segments,
416 std::optional<mlir::ArrayAttr> hasWaitDevnum,
417 mlir::acc::DeviceType deviceType) {
418 auto range =
419 getValuesFromSegments(arrayAttr: deviceTypeAttr, range: operands, segments, deviceType);
420 if (range.empty())
421 return range;
422 if (auto pos = findSegment(segments: *deviceTypeAttr, deviceType)) {
423 if (hasWaitDevnum && *hasWaitDevnum) {
424 auto boolAttr = mlir::dyn_cast<mlir::BoolAttr>(Val: (*hasWaitDevnum)[*pos]);
425 if (boolAttr.getValue())
426 return range.drop_front(n: 1); // first value is devnum
427 }
428 }
429 return range;
430}
431
432template <typename Op>
433static LogicalResult checkWaitAndAsyncConflict(Op op) {
434 for (uint32_t dtypeInt = 0; dtypeInt != acc::getMaxEnumValForDeviceType();
435 ++dtypeInt) {
436 auto dtype = static_cast<acc::DeviceType>(dtypeInt);
437
438 // The asyncOnly attribute represent the async clause without value.
439 // Therefore the attribute and operand cannot appear at the same time.
440 if (hasDeviceType(op.getAsyncOperandsDeviceType(), dtype) &&
441 op.hasAsyncOnly(dtype))
442 return op.emitError(
443 "asyncOnly attribute cannot appear with asyncOperand");
444
445 // The wait attribute represent the wait clause without values. Therefore
446 // the attribute and operands cannot appear at the same time.
447 if (hasDeviceType(op.getWaitOperandsDeviceType(), dtype) &&
448 op.hasWaitOnly(dtype))
449 return op.emitError("wait attribute cannot appear with waitOperands");
450 }
451 return success();
452}
453
454template <typename Op>
455static LogicalResult checkVarAndVarType(Op op) {
456 if (!op.getVar())
457 return op.emitError("must have var operand");
458
459 // A variable must have a type that is either pointer-like or mappable.
460 if (!mlir::isa<mlir::acc::PointerLikeType>(op.getVar().getType()) &&
461 !mlir::isa<mlir::acc::MappableType>(op.getVar().getType()))
462 return op.emitError("var must be mappable or pointer-like");
463
464 // When it is a pointer-like type, the varType must capture the target type.
465 if (mlir::isa<mlir::acc::PointerLikeType>(op.getVar().getType()) &&
466 op.getVarType() == op.getVar().getType())
467 return op.emitError("varType must capture the element type of var");
468
469 return success();
470}
471
472template <typename Op>
473static LogicalResult checkVarAndAccVar(Op op) {
474 if (op.getVar().getType() != op.getAccVar().getType())
475 return op.emitError("input and output types must match");
476
477 return success();
478}
479
480template <typename Op>
481static LogicalResult checkNoModifier(Op op) {
482 if (op.getModifiers() != acc::DataClauseModifier::none)
483 return op.emitError("no data clause modifiers are allowed");
484 return success();
485}
486
487template <typename Op>
488static LogicalResult
489checkValidModifier(Op op, acc::DataClauseModifier validModifiers) {
490 if (acc::bitEnumContainsAny(op.getModifiers(), ~validModifiers))
491 return op.emitError(
492 "invalid data clause modifiers: " +
493 acc::stringifyDataClauseModifier(symbol: op.getModifiers() & ~validModifiers));
494
495 return success();
496}
497
498template <typename OpT, typename RecipeOpT>
499static LogicalResult checkRecipe(OpT op, llvm::StringRef operandName) {
500 // Mappable types do not need a recipe because it is possible to generate one
501 // from its API. Reject reductions though because no API is available for them
502 // at this time.
503 if (mlir::acc::isMappableType(type: op.getVar().getType()) &&
504 !std::is_same_v<OpT, acc::ReductionOp>)
505 return success();
506
507 mlir::SymbolRefAttr operandRecipe = op.getRecipeAttr();
508 if (!operandRecipe)
509 return op->emitOpError() << "recipe expected for " << operandName;
510
511 auto decl =
512 SymbolTable::lookupNearestSymbolFrom<RecipeOpT>(op, operandRecipe);
513 if (!decl)
514 return op->emitOpError()
515 << "expected symbol reference " << operandRecipe << " to point to a "
516 << operandName << " declaration";
517 return success();
518}
519
520static ParseResult parseVar(mlir::OpAsmParser &parser,
521 OpAsmParser::UnresolvedOperand &var) {
522 // Either `var` or `varPtr` keyword is required.
523 if (failed(Result: parser.parseOptionalKeyword(keyword: "varPtr"))) {
524 if (failed(Result: parser.parseKeyword(keyword: "var")))
525 return failure();
526 }
527 if (failed(Result: parser.parseLParen()))
528 return failure();
529 if (failed(Result: parser.parseOperand(result&: var)))
530 return failure();
531
532 return success();
533}
534
535static void printVar(mlir::OpAsmPrinter &p, mlir::Operation *op,
536 mlir::Value var) {
537 if (mlir::isa<mlir::acc::PointerLikeType>(Val: var.getType()))
538 p << "varPtr(";
539 else
540 p << "var(";
541 p.printOperand(value: var);
542}
543
544static ParseResult parseAccVar(mlir::OpAsmParser &parser,
545 OpAsmParser::UnresolvedOperand &var,
546 mlir::Type &accVarType) {
547 // Either `accVar` or `accPtr` keyword is required.
548 if (failed(Result: parser.parseOptionalKeyword(keyword: "accPtr"))) {
549 if (failed(Result: parser.parseKeyword(keyword: "accVar")))
550 return failure();
551 }
552 if (failed(Result: parser.parseLParen()))
553 return failure();
554 if (failed(Result: parser.parseOperand(result&: var)))
555 return failure();
556 if (failed(Result: parser.parseColon()))
557 return failure();
558 if (failed(Result: parser.parseType(result&: accVarType)))
559 return failure();
560 if (failed(Result: parser.parseRParen()))
561 return failure();
562
563 return success();
564}
565
566static void printAccVar(mlir::OpAsmPrinter &p, mlir::Operation *op,
567 mlir::Value accVar, mlir::Type accVarType) {
568 if (mlir::isa<mlir::acc::PointerLikeType>(Val: accVar.getType()))
569 p << "accPtr(";
570 else
571 p << "accVar(";
572 p.printOperand(value: accVar);
573 p << " : ";
574 p.printType(type: accVarType);
575 p << ")";
576}
577
578static ParseResult parseVarPtrType(mlir::OpAsmParser &parser,
579 mlir::Type &varPtrType,
580 mlir::TypeAttr &varTypeAttr) {
581 if (failed(Result: parser.parseType(result&: varPtrType)))
582 return failure();
583 if (failed(Result: parser.parseRParen()))
584 return failure();
585
586 if (succeeded(Result: parser.parseOptionalKeyword(keyword: "varType"))) {
587 if (failed(Result: parser.parseLParen()))
588 return failure();
589 mlir::Type varType;
590 if (failed(Result: parser.parseType(result&: varType)))
591 return failure();
592 varTypeAttr = mlir::TypeAttr::get(type: varType);
593 if (failed(Result: parser.parseRParen()))
594 return failure();
595 } else {
596 // Set `varType` from the element type of the type of `varPtr`.
597 if (auto ptrTy = dyn_cast<acc::PointerLikeType>(Val&: varPtrType)) {
598 Type elementType = ptrTy.getElementType();
599 // Opaque pointers (e.g. !llvm.ptr) have no element type; fall back to
600 // using varPtrType itself so that the attribute is always valid.
601 varTypeAttr = mlir::TypeAttr::get(type: elementType ? elementType : varPtrType);
602 } else {
603 varTypeAttr = mlir::TypeAttr::get(type: varPtrType);
604 }
605 }
606
607 return success();
608}
609
610static void printVarPtrType(mlir::OpAsmPrinter &p, mlir::Operation *op,
611 mlir::Type varPtrType, mlir::TypeAttr varTypeAttr) {
612 p.printType(type: varPtrType);
613 p << ")";
614
615 // Print the `varType` only if it differs from the element type of
616 // `varPtr`'s type.
617 mlir::Type varType = varTypeAttr.getValue();
618 mlir::Type typeToCheckAgainst =
619 mlir::isa<mlir::acc::PointerLikeType>(Val: varPtrType)
620 ? mlir::cast<mlir::acc::PointerLikeType>(Val&: varPtrType).getElementType()
621 : varPtrType;
622 // Opaque pointers (e.g. !llvm.ptr) have no element type; use varPtrType as
623 // the baseline so that the inferred varType is not redundantly printed.
624 if (!typeToCheckAgainst)
625 typeToCheckAgainst = varPtrType;
626 if (typeToCheckAgainst != varType) {
627 p << " varType(";
628 p.printType(type: varType);
629 p << ")";
630 }
631}
632
633// A location cannot be parsed through the generic attribute directive: the
634// generated parser needs a concrete attribute class, while any of the location
635// attributes may appear here.
636static ParseResult parseSourceLocation(mlir::OpAsmParser &parser,
637 mlir::LocationAttr &locAttr) {
638 llvm::SMLoc attrLoc = parser.getCurrentLocation();
639 mlir::Attribute attr;
640 if (failed(Result: parser.parseAttribute(result&: attr)))
641 return failure();
642 locAttr = mlir::dyn_cast<mlir::LocationAttr>(Val&: attr);
643 if (!locAttr)
644 return parser.emitError(loc: attrLoc, message: "expected location attribute");
645 return success();
646}
647
648static void printSourceLocation(mlir::OpAsmPrinter &p, mlir::Operation *op,
649 mlir::LocationAttr locAttr) {
650 p.printAttribute(attr: locAttr);
651}
652
653static ParseResult parseRecipeSym(mlir::OpAsmParser &parser,
654 mlir::SymbolRefAttr &recipeAttr) {
655 if (failed(Result: parser.parseAttribute(result&: recipeAttr)))
656 return failure();
657 return success();
658}
659
660static void printRecipeSym(mlir::OpAsmPrinter &p, mlir::Operation *op,
661 mlir::SymbolRefAttr recipeAttr) {
662 p << recipeAttr;
663}
664
665static ParseResult parseDenseBoolArrayAttr(mlir::OpAsmParser &parser,
666 mlir::DenseBoolArrayAttr &attr) {
667 return parser.parseAttribute(result&: attr);
668}
669
670static void printDenseBoolArrayAttr(mlir::OpAsmPrinter &p, mlir::Operation *op,
671 mlir::DenseBoolArrayAttr attr) {
672 p << attr;
673}
674
675static ParseResult parseArrayAttr(mlir::OpAsmParser &parser,
676 mlir::ArrayAttr &attr) {
677 return parser.parseAttribute(result&: attr);
678}
679
680static void printArrayAttr(mlir::OpAsmPrinter &p, mlir::Operation *op,
681 mlir::ArrayAttr attr) {
682 p << attr;
683}
684
685//===----------------------------------------------------------------------===//
686// DataBoundsOp
687//===----------------------------------------------------------------------===//
688LogicalResult acc::DataBoundsOp::verify() {
689 auto extent = getExtent();
690 auto upperbound = getUpperbound();
691 if (!extent && !upperbound)
692 return emitError(message: "expected extent or upperbound.");
693 return success();
694}
695
696//===----------------------------------------------------------------------===//
697// PrivateOp
698//===----------------------------------------------------------------------===//
699LogicalResult acc::PrivateOp::verify() {
700 if (getDataClause() != acc::DataClause::acc_private)
701 return emitError(
702 message: "data clause associated with private operation must match its intent");
703 if (failed(Result: checkVarAndVarType(op: *this)))
704 return failure();
705 if (failed(Result: checkNoModifier(op: *this)))
706 return failure();
707 if (failed(
708 Result: checkRecipe<acc::PrivateOp, acc::PrivateRecipeOp>(op: *this, operandName: "private")))
709 return failure();
710 return success();
711}
712
713//===----------------------------------------------------------------------===//
714// FirstprivateOp
715//===----------------------------------------------------------------------===//
716LogicalResult acc::FirstprivateOp::verify() {
717 if (getDataClause() != acc::DataClause::acc_firstprivate)
718 return emitError(message: "data clause associated with firstprivate operation must "
719 "match its intent");
720 if (failed(Result: checkVarAndVarType(op: *this)))
721 return failure();
722 if (failed(Result: checkNoModifier(op: *this)))
723 return failure();
724 if (failed(Result: checkRecipe<acc::FirstprivateOp, acc::FirstprivateRecipeOp>(
725 op: *this, operandName: "firstprivate")))
726 return failure();
727 return success();
728}
729
730//===----------------------------------------------------------------------===//
731// ReductionOp
732//===----------------------------------------------------------------------===//
733LogicalResult acc::ReductionOp::verify() {
734 if (getDataClause() != acc::DataClause::acc_reduction)
735 return emitError(message: "data clause associated with reduction operation must "
736 "match its intent");
737 if (failed(Result: checkVarAndVarType(op: *this)))
738 return failure();
739 if (failed(Result: checkNoModifier(op: *this)))
740 return failure();
741 if (failed(Result: checkRecipe<acc::ReductionOp, acc::ReductionRecipeOp>(
742 op: *this, operandName: "reduction")))
743 return failure();
744 return success();
745}
746
747//===----------------------------------------------------------------------===//
748// DevicePtrOp
749//===----------------------------------------------------------------------===//
750LogicalResult acc::DevicePtrOp::verify() {
751 if (getDataClause() != acc::DataClause::acc_deviceptr)
752 return emitError(message: "data clause associated with deviceptr operation must "
753 "match its intent");
754 if (failed(Result: checkVarAndVarType(op: *this)))
755 return failure();
756 if (failed(Result: checkVarAndAccVar(op: *this)))
757 return failure();
758 if (failed(Result: checkNoModifier(op: *this)))
759 return failure();
760 return success();
761}
762
763//===----------------------------------------------------------------------===//
764// PresentOp
765//===----------------------------------------------------------------------===//
766LogicalResult acc::PresentOp::verify() {
767 if (getDataClause() != acc::DataClause::acc_present)
768 return emitError(
769 message: "data clause associated with present operation must match its intent");
770 if (failed(Result: checkVarAndVarType(op: *this)))
771 return failure();
772 if (failed(Result: checkVarAndAccVar(op: *this)))
773 return failure();
774 if (failed(Result: checkNoModifier(op: *this)))
775 return failure();
776 return success();
777}
778
779//===----------------------------------------------------------------------===//
780// CopyinOp
781//===----------------------------------------------------------------------===//
782LogicalResult acc::CopyinOp::verify() {
783 // Test for all clauses this operation can be decomposed from:
784 if (!getImplicit() && getDataClause() != acc::DataClause::acc_copyin &&
785 getDataClause() != acc::DataClause::acc_copyin_readonly &&
786 getDataClause() != acc::DataClause::acc_copy &&
787 getDataClause() != acc::DataClause::acc_reduction)
788 return emitError(
789 message: "data clause associated with copyin operation must match its intent"
790 " or specify original clause this operation was decomposed from");
791 if (failed(Result: checkVarAndVarType(op: *this)))
792 return failure();
793 if (failed(Result: checkVarAndAccVar(op: *this)))
794 return failure();
795 if (failed(Result: checkValidModifier(op: *this, validModifiers: acc::DataClauseModifier::readonly |
796 acc::DataClauseModifier::always |
797 acc::DataClauseModifier::capture)))
798 return failure();
799 return success();
800}
801
802bool acc::CopyinOp::isCopyinReadonly() {
803 return getDataClause() == acc::DataClause::acc_copyin_readonly ||
804 acc::bitEnumContainsAny(bits: getModifiers(),
805 bit: acc::DataClauseModifier::readonly);
806}
807
808//===----------------------------------------------------------------------===//
809// CreateOp
810//===----------------------------------------------------------------------===//
811LogicalResult acc::CreateOp::verify() {
812 // Test for all clauses this operation can be decomposed from:
813 if (getDataClause() != acc::DataClause::acc_create &&
814 getDataClause() != acc::DataClause::acc_create_zero &&
815 getDataClause() != acc::DataClause::acc_copyout &&
816 getDataClause() != acc::DataClause::acc_copyout_zero)
817 return emitError(
818 message: "data clause associated with create operation must match its intent"
819 " or specify original clause this operation was decomposed from");
820 if (failed(Result: checkVarAndVarType(op: *this)))
821 return failure();
822 if (failed(Result: checkVarAndAccVar(op: *this)))
823 return failure();
824 // this op is the entry part of copyout, so it also needs to allow all
825 // modifiers allowed on copyout.
826 if (failed(Result: checkValidModifier(op: *this, validModifiers: acc::DataClauseModifier::zero |
827 acc::DataClauseModifier::always |
828 acc::DataClauseModifier::capture)))
829 return failure();
830 return success();
831}
832
833bool acc::CreateOp::isCreateZero() {
834 // The zero modifier is encoded in the data clause.
835 return getDataClause() == acc::DataClause::acc_create_zero ||
836 getDataClause() == acc::DataClause::acc_copyout_zero ||
837 acc::bitEnumContainsAny(bits: getModifiers(), bit: acc::DataClauseModifier::zero);
838}
839
840//===----------------------------------------------------------------------===//
841// NoCreateOp
842//===----------------------------------------------------------------------===//
843LogicalResult acc::NoCreateOp::verify() {
844 if (getDataClause() != acc::DataClause::acc_no_create)
845 return emitError(message: "data clause associated with no_create operation must "
846 "match its intent");
847 if (failed(Result: checkVarAndVarType(op: *this)))
848 return failure();
849 if (failed(Result: checkVarAndAccVar(op: *this)))
850 return failure();
851 if (failed(Result: checkNoModifier(op: *this)))
852 return failure();
853 return success();
854}
855
856//===----------------------------------------------------------------------===//
857// AttachOp
858//===----------------------------------------------------------------------===//
859LogicalResult acc::AttachOp::verify() {
860 if (getDataClause() != acc::DataClause::acc_attach)
861 return emitError(
862 message: "data clause associated with attach operation must match its intent");
863 if (failed(Result: checkVarAndVarType(op: *this)))
864 return failure();
865 if (failed(Result: checkVarAndAccVar(op: *this)))
866 return failure();
867 if (failed(Result: checkNoModifier(op: *this)))
868 return failure();
869 return success();
870}
871
872//===----------------------------------------------------------------------===//
873// DeclareDeviceResidentOp
874//===----------------------------------------------------------------------===//
875
876LogicalResult acc::DeclareDeviceResidentOp::verify() {
877 if (getDataClause() != acc::DataClause::acc_declare_device_resident)
878 return emitError(message: "data clause associated with device_resident operation "
879 "must match its intent");
880 if (failed(Result: checkVarAndVarType(op: *this)))
881 return failure();
882 if (failed(Result: checkVarAndAccVar(op: *this)))
883 return failure();
884 if (failed(Result: checkNoModifier(op: *this)))
885 return failure();
886 return success();
887}
888
889//===----------------------------------------------------------------------===//
890// DeclareLinkOp
891//===----------------------------------------------------------------------===//
892
893LogicalResult acc::DeclareLinkOp::verify() {
894 if (getDataClause() != acc::DataClause::acc_declare_link)
895 return emitError(
896 message: "data clause associated with link operation must match its intent");
897 if (failed(Result: checkVarAndVarType(op: *this)))
898 return failure();
899 if (failed(Result: checkVarAndAccVar(op: *this)))
900 return failure();
901 if (failed(Result: checkNoModifier(op: *this)))
902 return failure();
903 return success();
904}
905
906//===----------------------------------------------------------------------===//
907// CopyoutOp
908//===----------------------------------------------------------------------===//
909LogicalResult acc::CopyoutOp::verify() {
910 // Test for all clauses this operation can be decomposed from:
911 if (getDataClause() != acc::DataClause::acc_copyout &&
912 getDataClause() != acc::DataClause::acc_copyout_zero &&
913 getDataClause() != acc::DataClause::acc_copy &&
914 getDataClause() != acc::DataClause::acc_reduction)
915 return emitError(
916 message: "data clause associated with copyout operation must match its intent"
917 " or specify original clause this operation was decomposed from");
918 if (!getVar() || !getAccVar())
919 return emitError(message: "must have both host and device pointers");
920 if (failed(Result: checkVarAndVarType(op: *this)))
921 return failure();
922 if (failed(Result: checkVarAndAccVar(op: *this)))
923 return failure();
924 if (failed(Result: checkValidModifier(op: *this, validModifiers: acc::DataClauseModifier::zero |
925 acc::DataClauseModifier::always |
926 acc::DataClauseModifier::capture)))
927 return failure();
928 return success();
929}
930
931bool acc::CopyoutOp::isCopyoutZero() {
932 return getDataClause() == acc::DataClause::acc_copyout_zero ||
933 acc::bitEnumContainsAny(bits: getModifiers(), bit: acc::DataClauseModifier::zero);
934}
935
936//===----------------------------------------------------------------------===//
937// DeleteOp
938//===----------------------------------------------------------------------===//
939LogicalResult acc::DeleteOp::verify() {
940 // Test for all clauses this operation can be decomposed from:
941 if (getDataClause() != acc::DataClause::acc_delete &&
942 getDataClause() != acc::DataClause::acc_create &&
943 getDataClause() != acc::DataClause::acc_create_zero &&
944 getDataClause() != acc::DataClause::acc_copyin &&
945 getDataClause() != acc::DataClause::acc_copyin_readonly &&
946 getDataClause() != acc::DataClause::acc_present &&
947 getDataClause() != acc::DataClause::acc_no_create &&
948 getDataClause() != acc::DataClause::acc_declare_device_resident &&
949 getDataClause() != acc::DataClause::acc_declare_link)
950 return emitError(
951 message: "data clause associated with delete operation must match its intent"
952 " or specify original clause this operation was decomposed from");
953 if (!getAccVar())
954 return emitError(message: "must have device pointer");
955 // This op is the exit part of copyin and create - thus allow all modifiers
956 // allowed on either case.
957 if (failed(Result: checkValidModifier(op: *this, validModifiers: acc::DataClauseModifier::zero |
958 acc::DataClauseModifier::readonly |
959 acc::DataClauseModifier::always |
960 acc::DataClauseModifier::capture)))
961 return failure();
962 return success();
963}
964
965//===----------------------------------------------------------------------===//
966// DetachOp
967//===----------------------------------------------------------------------===//
968LogicalResult acc::DetachOp::verify() {
969 // Test for all clauses this operation can be decomposed from:
970 if (getDataClause() != acc::DataClause::acc_detach &&
971 getDataClause() != acc::DataClause::acc_attach)
972 return emitError(
973 message: "data clause associated with detach operation must match its intent"
974 " or specify original clause this operation was decomposed from");
975 if (!getAccVar())
976 return emitError(message: "must have device pointer");
977 if (failed(Result: checkNoModifier(op: *this)))
978 return failure();
979 return success();
980}
981
982//===----------------------------------------------------------------------===//
983// HostOp
984//===----------------------------------------------------------------------===//
985LogicalResult acc::UpdateHostOp::verify() {
986 // Test for all clauses this operation can be decomposed from:
987 if (getDataClause() != acc::DataClause::acc_update_host &&
988 getDataClause() != acc::DataClause::acc_update_self)
989 return emitError(
990 message: "data clause associated with host operation must match its intent"
991 " or specify original clause this operation was decomposed from");
992 if (!getVar() || !getAccVar())
993 return emitError(message: "must have both host and device pointers");
994 if (failed(Result: checkVarAndVarType(op: *this)))
995 return failure();
996 if (failed(Result: checkVarAndAccVar(op: *this)))
997 return failure();
998 if (failed(Result: checkNoModifier(op: *this)))
999 return failure();
1000 return success();
1001}
1002
1003//===----------------------------------------------------------------------===//
1004// DeviceOp
1005//===----------------------------------------------------------------------===//
1006LogicalResult acc::UpdateDeviceOp::verify() {
1007 // Test for all clauses this operation can be decomposed from:
1008 if (getDataClause() != acc::DataClause::acc_update_device)
1009 return emitError(
1010 message: "data clause associated with device operation must match its intent"
1011 " or specify original clause this operation was decomposed from");
1012 if (failed(Result: checkVarAndVarType(op: *this)))
1013 return failure();
1014 if (failed(Result: checkVarAndAccVar(op: *this)))
1015 return failure();
1016 if (failed(Result: checkNoModifier(op: *this)))
1017 return failure();
1018 return success();
1019}
1020
1021//===----------------------------------------------------------------------===//
1022// UseDeviceOp
1023//===----------------------------------------------------------------------===//
1024LogicalResult acc::UseDeviceOp::verify() {
1025 // Test for all clauses this operation can be decomposed from:
1026 if (getDataClause() != acc::DataClause::acc_use_device)
1027 return emitError(
1028 message: "data clause associated with use_device operation must match its intent"
1029 " or specify original clause this operation was decomposed from");
1030 if (failed(Result: checkVarAndVarType(op: *this)))
1031 return failure();
1032 if (failed(Result: checkVarAndAccVar(op: *this)))
1033 return failure();
1034 if (failed(Result: checkNoModifier(op: *this)))
1035 return failure();
1036 return success();
1037}
1038
1039//===----------------------------------------------------------------------===//
1040// CacheOp
1041//===----------------------------------------------------------------------===//
1042LogicalResult acc::CacheOp::verify() {
1043 // Test for all clauses this operation can be decomposed from:
1044 if (getDataClause() != acc::DataClause::acc_cache &&
1045 getDataClause() != acc::DataClause::acc_cache_readonly)
1046 return emitError(
1047 message: "data clause associated with cache operation must match its intent"
1048 " or specify original clause this operation was decomposed from");
1049 if (failed(Result: checkVarAndVarType(op: *this)))
1050 return failure();
1051 if (failed(Result: checkVarAndAccVar(op: *this)))
1052 return failure();
1053 if (failed(Result: checkValidModifier(op: *this, validModifiers: acc::DataClauseModifier::readonly)))
1054 return failure();
1055 return success();
1056}
1057
1058bool acc::CacheOp::isCacheReadonly() {
1059 return getDataClause() == acc::DataClause::acc_cache_readonly ||
1060 acc::bitEnumContainsAny(bits: getModifiers(),
1061 bit: acc::DataClauseModifier::readonly);
1062}
1063
1064//===----------------------------------------------------------------------===//
1065// Data entry/exit operations - getEffects implementations
1066//===----------------------------------------------------------------------===//
1067
1068// This function returns true iff the given operation is enclosed
1069// in any ACC_COMPUTE_CONSTRUCT_OPS operation.
1070// It is quite alike acc::getEnclosingComputeOp() utility,
1071// but we cannot use it here.
1072static bool isEnclosedIntoComputeOp(mlir::Operation *op) {
1073 return op->getParentOfType<ACC_COMPUTE_CONSTRUCT_OPS>();
1074}
1075
1076/// Helper to add an effect on an operand, referenced by its mutable range.
1077template <typename EffectTy>
1078static void addOperandEffect(
1079 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
1080 &effects,
1081 MutableOperandRange operand) {
1082 for (unsigned i = 0, e = operand.size(); i < e; ++i)
1083 effects.emplace_back(EffectTy::get(), &operand[i]);
1084}
1085
1086/// Helper to add an effect on a result value.
1087template <typename EffectTy>
1088static void addResultEffect(
1089 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
1090 &effects,
1091 Value result) {
1092 effects.emplace_back(EffectTy::get(), mlir::cast<mlir::OpResult>(Val&: result));
1093}
1094
1095// PrivateOp: accVar result write.
1096void acc::PrivateOp::getEffects(
1097 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
1098 &effects) {
1099 // If acc.private is enclosed into a compute operation,
1100 // then it denotes the device side privatization, hence
1101 // it does not access the CurrentDeviceIdResource.
1102 if (!isEnclosedIntoComputeOp(op: getOperation()))
1103 effects.emplace_back(Args: MemoryEffects::Read::get(),
1104 Args: acc::CurrentDeviceIdResource::get());
1105 // TODO: should this be MemoryEffects::Allocate?
1106 addResultEffect<MemoryEffects::Write>(effects, result: getAccVar());
1107}
1108
1109// FirstprivateOp: var read, accVar result write.
1110void acc::FirstprivateOp::getEffects(
1111 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
1112 &effects) {
1113 // If acc.firstprivate is enclosed into a compute operation,
1114 // then it denotes the device side privatization, hence
1115 // it does not access the CurrentDeviceIdResource.
1116 if (!isEnclosedIntoComputeOp(op: getOperation()))
1117 effects.emplace_back(Args: MemoryEffects::Read::get(),
1118 Args: acc::CurrentDeviceIdResource::get());
1119 addOperandEffect<MemoryEffects::Read>(effects, operand: getVarMutable());
1120 addResultEffect<MemoryEffects::Write>(effects, result: getAccVar());
1121}
1122
1123// ReductionOp: var read, accVar result write.
1124void acc::ReductionOp::getEffects(
1125 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
1126 &effects) {
1127 // If acc.reduction is enclosed into a compute operation,
1128 // then it denotes the device side reduction, hence
1129 // it does not access the CurrentDeviceIdResource.
1130 if (!isEnclosedIntoComputeOp(op: getOperation()))
1131 effects.emplace_back(Args: MemoryEffects::Read::get(),
1132 Args: acc::CurrentDeviceIdResource::get());
1133 addOperandEffect<MemoryEffects::Read>(effects, operand: getVarMutable());
1134 addResultEffect<MemoryEffects::Write>(effects, result: getAccVar());
1135}
1136
1137// DevicePtrOp: RuntimeCounters read.
1138void acc::DevicePtrOp::getEffects(
1139 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
1140 &effects) {
1141 effects.emplace_back(Args: MemoryEffects::Read::get(), Args: acc::RuntimeCounters::get());
1142 effects.emplace_back(Args: MemoryEffects::Read::get(),
1143 Args: acc::CurrentDeviceIdResource::get());
1144}
1145
1146// PresentOp: RuntimeCounters read+write.
1147void acc::PresentOp::getEffects(
1148 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
1149 &effects) {
1150 effects.emplace_back(Args: MemoryEffects::Read::get(), Args: acc::RuntimeCounters::get());
1151 effects.emplace_back(Args: MemoryEffects::Write::get(),
1152 Args: acc::RuntimeCounters::get());
1153 effects.emplace_back(Args: MemoryEffects::Read::get(),
1154 Args: acc::CurrentDeviceIdResource::get());
1155}
1156
1157// CopyinOp: RuntimeCounters read+write, var read, accVar result write.
1158void acc::CopyinOp::getEffects(
1159 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
1160 &effects) {
1161 effects.emplace_back(Args: MemoryEffects::Read::get(), Args: acc::RuntimeCounters::get());
1162 effects.emplace_back(Args: MemoryEffects::Write::get(),
1163 Args: acc::RuntimeCounters::get());
1164 effects.emplace_back(Args: MemoryEffects::Read::get(),
1165 Args: acc::CurrentDeviceIdResource::get());
1166 addOperandEffect<MemoryEffects::Read>(effects, operand: getVarMutable());
1167 addResultEffect<MemoryEffects::Write>(effects, result: getAccVar());
1168}
1169
1170// CreateOp: RuntimeCounters read+write, accVar result write.
1171void acc::CreateOp::getEffects(
1172 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
1173 &effects) {
1174 effects.emplace_back(Args: MemoryEffects::Read::get(), Args: acc::RuntimeCounters::get());
1175 effects.emplace_back(Args: MemoryEffects::Write::get(),
1176 Args: acc::RuntimeCounters::get());
1177 effects.emplace_back(Args: MemoryEffects::Read::get(),
1178 Args: acc::CurrentDeviceIdResource::get());
1179 // TODO: should this be MemoryEffects::Allocate?
1180 addResultEffect<MemoryEffects::Write>(effects, result: getAccVar());
1181}
1182
1183// NoCreateOp: RuntimeCounters read+write.
1184void acc::NoCreateOp::getEffects(
1185 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
1186 &effects) {
1187 effects.emplace_back(Args: MemoryEffects::Read::get(), Args: acc::RuntimeCounters::get());
1188 effects.emplace_back(Args: MemoryEffects::Write::get(),
1189 Args: acc::RuntimeCounters::get());
1190 effects.emplace_back(Args: MemoryEffects::Read::get(),
1191 Args: acc::CurrentDeviceIdResource::get());
1192}
1193
1194// AttachOp: RuntimeCounters read+write, var read.
1195void acc::AttachOp::getEffects(
1196 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
1197 &effects) {
1198 effects.emplace_back(Args: MemoryEffects::Read::get(), Args: acc::RuntimeCounters::get());
1199 effects.emplace_back(Args: MemoryEffects::Write::get(),
1200 Args: acc::RuntimeCounters::get());
1201 effects.emplace_back(Args: MemoryEffects::Read::get(),
1202 Args: acc::CurrentDeviceIdResource::get());
1203 // TODO: should we also add MemoryEffects::Write?
1204 addOperandEffect<MemoryEffects::Read>(effects, operand: getVarMutable());
1205}
1206
1207// GetDevicePtrOp: RuntimeCounters read.
1208void acc::GetDevicePtrOp::getEffects(
1209 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
1210 &effects) {
1211 effects.emplace_back(Args: MemoryEffects::Read::get(), Args: acc::RuntimeCounters::get());
1212 effects.emplace_back(Args: MemoryEffects::Read::get(),
1213 Args: acc::CurrentDeviceIdResource::get());
1214}
1215
1216// UpdateDeviceOp: var read, accVar result write.
1217void acc::UpdateDeviceOp::getEffects(
1218 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
1219 &effects) {
1220 effects.emplace_back(Args: MemoryEffects::Read::get(),
1221 Args: acc::CurrentDeviceIdResource::get());
1222 addOperandEffect<MemoryEffects::Read>(effects, operand: getVarMutable());
1223 addResultEffect<MemoryEffects::Write>(effects, result: getAccVar());
1224}
1225
1226// UseDeviceOp: RuntimeCounters read.
1227void acc::UseDeviceOp::getEffects(
1228 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
1229 &effects) {
1230 effects.emplace_back(Args: MemoryEffects::Read::get(), Args: acc::RuntimeCounters::get());
1231 effects.emplace_back(Args: MemoryEffects::Read::get(),
1232 Args: acc::CurrentDeviceIdResource::get());
1233}
1234
1235// DeclareDeviceResidentOp: RuntimeCounters write, var read.
1236void acc::DeclareDeviceResidentOp::getEffects(
1237 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
1238 &effects) {
1239 effects.emplace_back(Args: MemoryEffects::Write::get(),
1240 Args: acc::RuntimeCounters::get());
1241 effects.emplace_back(Args: MemoryEffects::Read::get(),
1242 Args: acc::CurrentDeviceIdResource::get());
1243 addOperandEffect<MemoryEffects::Read>(effects, operand: getVarMutable());
1244}
1245
1246// DeclareLinkOp: RuntimeCounters write, var read.
1247void acc::DeclareLinkOp::getEffects(
1248 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
1249 &effects) {
1250 effects.emplace_back(Args: MemoryEffects::Write::get(),
1251 Args: acc::RuntimeCounters::get());
1252 effects.emplace_back(Args: MemoryEffects::Read::get(),
1253 Args: acc::CurrentDeviceIdResource::get());
1254 addOperandEffect<MemoryEffects::Read>(effects, operand: getVarMutable());
1255}
1256
1257// CacheOp: NoMemoryEffect
1258void acc::CacheOp::getEffects(
1259 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
1260 &effects) {}
1261
1262// CopyoutOp: RuntimeCounters read+write, accVar read, var write.
1263void acc::CopyoutOp::getEffects(
1264 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
1265 &effects) {
1266 effects.emplace_back(Args: MemoryEffects::Read::get(), Args: acc::RuntimeCounters::get());
1267 effects.emplace_back(Args: MemoryEffects::Write::get(),
1268 Args: acc::RuntimeCounters::get());
1269 effects.emplace_back(Args: MemoryEffects::Read::get(),
1270 Args: acc::CurrentDeviceIdResource::get());
1271 addOperandEffect<MemoryEffects::Read>(effects, operand: getAccVarMutable());
1272 addOperandEffect<MemoryEffects::Write>(effects, operand: getVarMutable());
1273}
1274
1275// DeleteOp: RuntimeCounters read+write, accVar read.
1276void acc::DeleteOp::getEffects(
1277 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
1278 &effects) {
1279 effects.emplace_back(Args: MemoryEffects::Read::get(), Args: acc::RuntimeCounters::get());
1280 effects.emplace_back(Args: MemoryEffects::Write::get(),
1281 Args: acc::RuntimeCounters::get());
1282 effects.emplace_back(Args: MemoryEffects::Read::get(),
1283 Args: acc::CurrentDeviceIdResource::get());
1284 addOperandEffect<MemoryEffects::Read>(effects, operand: getAccVarMutable());
1285}
1286
1287// DetachOp: RuntimeCounters read+write, accVar read.
1288void acc::DetachOp::getEffects(
1289 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
1290 &effects) {
1291 effects.emplace_back(Args: MemoryEffects::Read::get(), Args: acc::RuntimeCounters::get());
1292 effects.emplace_back(Args: MemoryEffects::Write::get(),
1293 Args: acc::RuntimeCounters::get());
1294 effects.emplace_back(Args: MemoryEffects::Read::get(),
1295 Args: acc::CurrentDeviceIdResource::get());
1296 addOperandEffect<MemoryEffects::Read>(effects, operand: getAccVarMutable());
1297}
1298
1299// UpdateHostOp: RuntimeCounters read+write, accVar read, var write.
1300void acc::UpdateHostOp::getEffects(
1301 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
1302 &effects) {
1303 effects.emplace_back(Args: MemoryEffects::Read::get(), Args: acc::RuntimeCounters::get());
1304 effects.emplace_back(Args: MemoryEffects::Write::get(),
1305 Args: acc::RuntimeCounters::get());
1306 effects.emplace_back(Args: MemoryEffects::Read::get(),
1307 Args: acc::CurrentDeviceIdResource::get());
1308 addOperandEffect<MemoryEffects::Read>(effects, operand: getAccVarMutable());
1309 addOperandEffect<MemoryEffects::Write>(effects, operand: getVarMutable());
1310}
1311
1312namespace {
1313/// Pattern to remove operation without region that have constant false `ifCond`
1314/// and remove the condition from the operation if the `ifCond` is a true
1315/// constant.
1316template <typename OpTy>
1317struct RemoveConstantIfCondition : public OpRewritePattern<OpTy> {
1318 using OpRewritePattern<OpTy>::OpRewritePattern;
1319
1320 LogicalResult matchAndRewrite(OpTy op,
1321 PatternRewriter &rewriter) const override {
1322 // Early return if there is no condition.
1323 Value ifCond = op.getIfCond();
1324 if (!ifCond)
1325 return failure();
1326
1327 IntegerAttr constAttr;
1328 if (!matchPattern(value: ifCond, pattern: m_Constant(bind_value: &constAttr)))
1329 return failure();
1330 if (constAttr.getInt())
1331 rewriter.modifyOpInPlace(op, [&]() { op.getIfCondMutable().erase(0); });
1332 else
1333 rewriter.eraseOp(op);
1334
1335 return success();
1336 }
1337};
1338
1339/// Replaces the given op with the contents of the given single-block region,
1340/// using the operands of the block terminator to replace operation results.
1341static void replaceOpWithRegion(PatternRewriter &rewriter, Operation *op,
1342 Region &region, ValueRange blockArgs = {}) {
1343 assert(region.hasOneBlock() && "expected single-block region");
1344 Block *block = &region.front();
1345 Operation *terminator = block->getTerminator();
1346 ValueRange results = terminator->getOperands();
1347 rewriter.inlineBlockBefore(source: block, op, argValues: blockArgs);
1348 rewriter.replaceOp(op, newValues: results);
1349 rewriter.eraseOp(op: terminator);
1350}
1351
1352/// Pattern to remove operation with region that have constant false `ifCond`
1353/// and remove the condition from the operation if the `ifCond` is constant
1354/// true.
1355template <typename OpTy>
1356struct RemoveConstantIfConditionWithRegion : public OpRewritePattern<OpTy> {
1357 using OpRewritePattern<OpTy>::OpRewritePattern;
1358
1359 LogicalResult matchAndRewrite(OpTy op,
1360 PatternRewriter &rewriter) const override {
1361 // Early return if there is no condition.
1362 Value ifCond = op.getIfCond();
1363 if (!ifCond)
1364 return failure();
1365
1366 IntegerAttr constAttr;
1367 if (!matchPattern(value: ifCond, pattern: m_Constant(bind_value: &constAttr)))
1368 return failure();
1369 if (constAttr.getInt()) {
1370 rewriter.modifyOpInPlace(op, [&]() { op.getIfCondMutable().erase(0); });
1371 return success();
1372 }
1373
1374 // With a constant false condition the construct has no effect, so it is
1375 // replaced by its body. An empty region has no body, so the op is simply
1376 // erased.
1377 Region &region = op.getRegion();
1378 if (region.empty()) {
1379 if (op->getNumResults() != 0)
1380 return rewriter.notifyMatchFailure(op, "empty region with results");
1381 rewriter.eraseOp(op);
1382 return success();
1383 }
1384 if (!region.hasOneBlock())
1385 return rewriter.notifyMatchFailure(op, "expected single-block region");
1386
1387 replaceOpWithRegion(rewriter, op, region);
1388 return success();
1389 }
1390};
1391
1392//===----------------------------------------------------------------------===//
1393// Recipe Region Helpers
1394//===----------------------------------------------------------------------===//
1395
1396/// Create and populate an init region for privatization recipes.
1397/// Returns success if the region is populated, failure otherwise.
1398/// Sets needsFree to indicate if the allocated memory requires deallocation.
1399/// The `hostVar` is the original host variable used to derive
1400/// language-specific metadata via `genPrivateVariableInfo`.
1401/// The `varInfo` output parameter is set to the variable info produced.
1402static LogicalResult createInitRegion(OpBuilder &builder, Location loc,
1403 Region &initRegion, Value hostVar,
1404 StringRef varName, ValueRange bounds,
1405 bool &needsFree,
1406 acc::VariableInfoAttr &varInfo,
1407 SmallVectorImpl<Value> &destroyValues) {
1408 Type varType = hostVar.getType();
1409
1410 // Create init block with arguments: original value + bounds
1411 SmallVector<Type> argTypes{varType};
1412 SmallVector<Location> argLocs{loc};
1413 for (Value bound : bounds) {
1414 argTypes.push_back(Elt: bound.getType());
1415 argLocs.push_back(Elt: loc);
1416 }
1417
1418 Block *initBlock = builder.createBlock(parent: &initRegion);
1419 initBlock->addArguments(types: argTypes, locs: argLocs);
1420 builder.setInsertionPointToStart(initBlock);
1421
1422 Value privatizedValue;
1423
1424 // Get the block argument that represents the original variable
1425 Value blockArgVar = initBlock->getArgument(i: 0);
1426
1427 // Generate init region body based on variable type
1428 if (isa<MappableType>(Val: varType)) {
1429 auto mappableTy = cast<MappableType>(Val&: varType);
1430 auto typedVar = cast<TypedValue<MappableType>>(Val&: blockArgVar);
1431 auto typedHostVar = cast<TypedValue<MappableType>>(Val&: hostVar);
1432 varInfo = mappableTy.genPrivateVariableInfo(var: typedHostVar);
1433 privatizedValue =
1434 mappableTy.generatePrivateInit(builder, loc, var: typedVar, varName, extents: bounds,
1435 initVal: {}, varInfo, needsDestroy&: needsFree, destroyValues);
1436 if (!privatizedValue)
1437 return failure();
1438 } else {
1439 assert(isa<PointerLikeType>(varType) && "Expected PointerLikeType");
1440 auto pointerLikeTy = cast<PointerLikeType>(Val&: varType);
1441 // Use PointerLikeType's allocation API with the block argument
1442 privatizedValue = pointerLikeTy.genAllocate(builder, loc, varName, varType,
1443 originalVar: blockArgVar, needsFree);
1444 if (!privatizedValue)
1445 return failure();
1446 }
1447
1448 // Add yield operation to init block
1449 SmallVector<Value> initResults{privatizedValue};
1450 initResults.append(RHS: destroyValues);
1451 acc::YieldOp::create(builder, location: loc, operands: initResults);
1452
1453 return success();
1454}
1455
1456/// Create and populate a copy region for firstprivate recipes.
1457/// Returns success if the region is populated, failure otherwise.
1458/// `varInfo` must be the attribute produced by `createInitRegion` for
1459/// `MappableType` (it is unused for `PointerLikeType` copy paths).
1460static LogicalResult createCopyRegion(OpBuilder &builder, Location loc,
1461 Region &copyRegion, Type varType,
1462 ValueRange bounds,
1463 acc::VariableInfoAttr varInfo) {
1464 // Create copy block with arguments: original value + privatized value +
1465 // bounds
1466 SmallVector<Type> copyArgTypes{varType, varType};
1467 SmallVector<Location> copyArgLocs{loc, loc};
1468 for (Value bound : bounds) {
1469 copyArgTypes.push_back(Elt: bound.getType());
1470 copyArgLocs.push_back(Elt: loc);
1471 }
1472
1473 Block *copyBlock = builder.createBlock(parent: &copyRegion);
1474 copyBlock->addArguments(types: copyArgTypes, locs: copyArgLocs);
1475 builder.setInsertionPointToStart(copyBlock);
1476
1477 Value originalArg = copyBlock->getArgument(i: 0);
1478 Value privatizedArg = copyBlock->getArgument(i: 1);
1479
1480 if (isa<MappableType>(Val: varType)) {
1481 auto mappableTy = cast<MappableType>(Val&: varType);
1482 // generateCopy(src, dest): copy from original (arg0) into privatized
1483 // (arg1).
1484 if (!mappableTy.generateCopy(
1485 builder, loc, from: cast<TypedValue<MappableType>>(Val&: originalArg),
1486 to: cast<TypedValue<MappableType>>(Val&: privatizedArg), bounds, varInfo))
1487 return failure();
1488 } else {
1489 assert(isa<PointerLikeType>(varType) && "Expected PointerLikeType");
1490 auto pointerLikeTy = cast<PointerLikeType>(Val&: varType);
1491 if (!pointerLikeTy.genCopy(
1492 builder, loc, destination: cast<TypedValue<PointerLikeType>>(Val&: privatizedArg),
1493 source: cast<TypedValue<PointerLikeType>>(Val&: originalArg), varType))
1494 return failure();
1495 }
1496
1497 // Add terminator to copy block
1498 acc::TerminatorOp::create(builder, location: loc);
1499
1500 return success();
1501}
1502
1503/// Create and populate a destroy region for privatization recipes.
1504/// Returns success if the region is populated, failure otherwise.
1505/// The `varInfo` carries language-specific metadata produced by
1506/// `createInitRegion`.
1507static LogicalResult
1508createDestroyRegion(OpBuilder &builder, Location loc, Region &destroyRegion,
1509 Type varType, Value allocRes, ValueRange destroyValues,
1510 ValueRange bounds, acc::VariableInfoAttr varInfo) {
1511 // Create destroy block with arguments: original value + privatized value +
1512 // values preserved for destruction + bounds.
1513 SmallVector<Type> destroyArgTypes{varType, varType};
1514 SmallVector<Location> destroyArgLocs{loc, loc};
1515 for (Value destroyValue : destroyValues) {
1516 destroyArgTypes.push_back(Elt: destroyValue.getType());
1517 destroyArgLocs.push_back(Elt: loc);
1518 }
1519 for (Value bound : bounds) {
1520 destroyArgTypes.push_back(Elt: bound.getType());
1521 destroyArgLocs.push_back(Elt: loc);
1522 }
1523
1524 Block *destroyBlock = builder.createBlock(parent: &destroyRegion);
1525 destroyBlock->addArguments(types: destroyArgTypes, locs: destroyArgLocs);
1526 builder.setInsertionPointToStart(destroyBlock);
1527
1528 auto varToFree =
1529 cast<TypedValue<PointerLikeType>>(Val: destroyBlock->getArgument(i: 1));
1530 if (isa<MappableType>(Val: varType)) {
1531 auto mappableTy = cast<MappableType>(Val&: varType);
1532 ValueRange destroyArgs =
1533 destroyBlock->getArguments().slice(N: 2, M: destroyValues.size());
1534 ValueRange destroyBounds =
1535 destroyBlock->getArguments().drop_front(N: 2 + destroyValues.size());
1536 if (!mappableTy.generatePrivateDestroy(builder, loc, privatized: varToFree, destroyValues: destroyArgs,
1537 extents: destroyBounds, varInfo))
1538 return failure();
1539 } else {
1540 assert(isa<PointerLikeType>(varType) && "Expected PointerLikeType");
1541 auto pointerLikeTy = cast<PointerLikeType>(Val&: varType);
1542 if (!pointerLikeTy.genFree(builder, loc, varToFree, allocRes, varType))
1543 return failure();
1544 }
1545
1546 acc::TerminatorOp::create(builder, location: loc);
1547 return success();
1548}
1549} // namespace
1550
1551//===----------------------------------------------------------------------===//
1552// PrivateRecipeOp
1553//===----------------------------------------------------------------------===//
1554
1555static LogicalResult verifyInitLikeSingleArgRegion(
1556 Operation *op, Region &region, StringRef regionType, StringRef regionName,
1557 Type type, bool verifyYield, bool optional = false) {
1558 if (optional && region.empty())
1559 return success();
1560
1561 if (region.empty())
1562 return op->emitOpError() << "expects non-empty " << regionName << " region";
1563 Block &firstBlock = region.front();
1564 if (firstBlock.getNumArguments() < 1 ||
1565 firstBlock.getArgument(i: 0).getType() != type)
1566 return op->emitOpError() << "expects " << regionName
1567 << " region first "
1568 "argument of the "
1569 << regionType << " type";
1570
1571 if (verifyYield) {
1572 for (YieldOp yieldOp : region.getOps<acc::YieldOp>()) {
1573 if (yieldOp.getOperands().size() != 1 ||
1574 yieldOp.getOperands().getTypes()[0] != type)
1575 return op->emitOpError() << "expects " << regionName
1576 << " region to "
1577 "yield a value of the "
1578 << regionType << " type";
1579 }
1580 }
1581 return success();
1582}
1583
1584LogicalResult acc::PrivateRecipeOp::verifyRegions() {
1585 if (failed(Result: verifyInitLikeSingleArgRegion(op: *this, region&: getInitRegion(),
1586 regionType: "privatization", regionName: "init", type: getType(),
1587 /*verifyYield=*/false)))
1588 return failure();
1589 if (failed(Result: verifyInitLikeSingleArgRegion(
1590 op: *this, region&: getDestroyRegion(), regionType: "privatization", regionName: "destroy", type: getType(),
1591 /*verifyYield=*/false, /*optional=*/true)))
1592 return failure();
1593 return success();
1594}
1595
1596std::optional<PrivateRecipeOp>
1597PrivateRecipeOp::createAndPopulate(OpBuilder &builder, Location loc,
1598 StringRef recipeName, Value hostVar,
1599 StringRef varName, ValueRange bounds) {
1600 Type varType = hostVar.getType();
1601
1602 // First, validate that we can handle this variable type
1603 bool isMappable = isa<MappableType>(Val: varType);
1604 bool isPointerLike = isa<PointerLikeType>(Val: varType);
1605
1606 // Unsupported type
1607 if (!isMappable && !isPointerLike)
1608 return std::nullopt;
1609
1610 OpBuilder::InsertionGuard guard(builder);
1611
1612 // Create the recipe operation first so regions have proper parent context
1613 auto recipe = PrivateRecipeOp::create(builder, location: loc, sym_name: recipeName,
1614 /*sym_visibility=*/nullptr, type: varType);
1615
1616 // Populate the init region
1617 bool needsFree = false;
1618 acc::VariableInfoAttr varInfo;
1619 SmallVector<Value> destroyValues;
1620 if (failed(Result: createInitRegion(builder, loc, initRegion&: recipe.getInitRegion(), hostVar,
1621 varName, bounds, needsFree, varInfo,
1622 destroyValues))) {
1623 recipe.erase();
1624 return std::nullopt;
1625 }
1626
1627 // Only create destroy region if the allocation needs deallocation
1628 if (needsFree) {
1629 // Extract the allocated value from the init block's yield operation
1630 auto yieldOp =
1631 cast<acc::YieldOp>(Val: recipe.getInitRegion().front().getTerminator());
1632 Value allocRes = yieldOp.getOperand(i: 0);
1633
1634 if (failed(Result: createDestroyRegion(builder, loc, destroyRegion&: recipe.getDestroyRegion(),
1635 varType, allocRes, destroyValues, bounds,
1636 varInfo))) {
1637 recipe.erase();
1638 return std::nullopt;
1639 }
1640 }
1641
1642 return recipe;
1643}
1644
1645std::optional<PrivateRecipeOp>
1646PrivateRecipeOp::createAndPopulate(OpBuilder &builder, Location loc,
1647 StringRef recipeName,
1648 FirstprivateRecipeOp firstprivRecipe) {
1649 // Create the private.recipe op with the same type as the firstprivate.recipe.
1650 OpBuilder::InsertionGuard guard(builder);
1651 auto varType = firstprivRecipe.getType();
1652 auto recipe = PrivateRecipeOp::create(builder, location: loc, sym_name: recipeName,
1653 /*sym_visibility=*/nullptr, type: varType);
1654
1655 // Clone the init region
1656 IRMapping mapping;
1657 firstprivRecipe.getInitRegion().cloneInto(dest: &recipe.getInitRegion(), mapper&: mapping);
1658
1659 // Clone destroy region if the firstprivate.recipe has one.
1660 if (!firstprivRecipe.getDestroyRegion().empty()) {
1661 IRMapping mapping;
1662 firstprivRecipe.getDestroyRegion().cloneInto(dest: &recipe.getDestroyRegion(),
1663 mapper&: mapping);
1664 }
1665 return recipe;
1666}
1667
1668//===----------------------------------------------------------------------===//
1669// FirstprivateRecipeOp
1670//===----------------------------------------------------------------------===//
1671
1672LogicalResult acc::FirstprivateRecipeOp::verifyRegions() {
1673 if (failed(Result: verifyInitLikeSingleArgRegion(op: *this, region&: getInitRegion(),
1674 regionType: "privatization", regionName: "init", type: getType(),
1675 /*verifyYield=*/false)))
1676 return failure();
1677
1678 if (getCopyRegion().empty())
1679 return emitOpError() << "expects non-empty copy region";
1680
1681 Block &firstBlock = getCopyRegion().front();
1682 if (firstBlock.getNumArguments() < 2 ||
1683 firstBlock.getArgument(i: 0).getType() != getType())
1684 return emitOpError() << "expects copy region with two arguments of the "
1685 "privatization type";
1686
1687 if (getDestroyRegion().empty())
1688 return success();
1689
1690 if (failed(Result: verifyInitLikeSingleArgRegion(op: *this, region&: getDestroyRegion(),
1691 regionType: "privatization", regionName: "destroy",
1692 type: getType(), /*verifyYield=*/false)))
1693 return failure();
1694
1695 return success();
1696}
1697
1698std::optional<FirstprivateRecipeOp>
1699FirstprivateRecipeOp::createAndPopulate(OpBuilder &builder, Location loc,
1700 StringRef recipeName, Value hostVar,
1701 StringRef varName, ValueRange bounds) {
1702 Type varType = hostVar.getType();
1703
1704 // First, validate that we can handle this variable type
1705 bool isMappable = isa<MappableType>(Val: varType);
1706 bool isPointerLike = isa<PointerLikeType>(Val: varType);
1707
1708 // Unsupported type
1709 if (!isMappable && !isPointerLike)
1710 return std::nullopt;
1711
1712 OpBuilder::InsertionGuard guard(builder);
1713
1714 // Create the recipe operation first so regions have proper parent context
1715 auto recipe = FirstprivateRecipeOp::create(
1716 builder, location: loc, sym_name: recipeName, /*sym_visibility=*/nullptr, type: varType);
1717
1718 // Populate the init region
1719 bool needsFree = false;
1720 // Filled by createInitRegion for mappable variables (genPrivateVariableInfo);
1721 // then passed through to copy/destroy so generateCopy /
1722 // generatePrivateDestroy receive the same metadata as generatePrivateInit.
1723 acc::VariableInfoAttr varInfo;
1724 SmallVector<Value> destroyValues;
1725 if (failed(Result: createInitRegion(builder, loc, initRegion&: recipe.getInitRegion(), hostVar,
1726 varName, bounds, needsFree, varInfo,
1727 destroyValues))) {
1728 recipe.erase();
1729 return std::nullopt;
1730 }
1731
1732 // Populate the copy region (uses varInfo for MappableType::generateCopy).
1733 if (failed(Result: createCopyRegion(builder, loc, copyRegion&: recipe.getCopyRegion(), varType,
1734 bounds, varInfo))) {
1735 recipe.erase();
1736 return std::nullopt;
1737 }
1738
1739 // Only create destroy region if the allocation needs deallocation
1740 if (needsFree) {
1741 // Extract the allocated value from the init block's yield operation
1742 auto yieldOp =
1743 cast<acc::YieldOp>(Val: recipe.getInitRegion().front().getTerminator());
1744 Value allocRes = yieldOp.getOperand(i: 0);
1745
1746 if (failed(Result: createDestroyRegion(builder, loc, destroyRegion&: recipe.getDestroyRegion(),
1747 varType, allocRes, destroyValues, bounds,
1748 varInfo))) {
1749 recipe.erase();
1750 return std::nullopt;
1751 }
1752 }
1753
1754 return recipe;
1755}
1756
1757//===----------------------------------------------------------------------===//
1758// ReductionRecipeOp
1759//===----------------------------------------------------------------------===//
1760
1761LogicalResult acc::ReductionRecipeOp::verifyRegions() {
1762 if (failed(Result: verifyInitLikeSingleArgRegion(op: *this, region&: getInitRegion(), regionType: "reduction",
1763 regionName: "init", type: getType(),
1764 /*verifyYield=*/false)))
1765 return failure();
1766
1767 if (getCombinerRegion().empty())
1768 return emitOpError() << "expects non-empty combiner region";
1769
1770 Block &reductionBlock = getCombinerRegion().front();
1771 if (reductionBlock.getNumArguments() < 2 ||
1772 reductionBlock.getArgument(i: 0).getType() != getType() ||
1773 reductionBlock.getArgument(i: 1).getType() != getType())
1774 return emitOpError() << "expects combiner region with the first two "
1775 << "arguments of the reduction type";
1776
1777 for (YieldOp yieldOp : getCombinerRegion().getOps<YieldOp>()) {
1778 if (yieldOp.getOperands().size() != 1 ||
1779 yieldOp.getOperands().getTypes()[0] != getType())
1780 return emitOpError() << "expects combiner region to yield a value "
1781 "of the reduction type";
1782 }
1783
1784 return success();
1785}
1786
1787//===----------------------------------------------------------------------===//
1788// ParallelOp
1789//===----------------------------------------------------------------------===//
1790
1791/// Check dataOperands for acc.parallel, acc.serial and acc.kernels.
1792template <typename Op>
1793static LogicalResult checkDataOperands(Op op,
1794 const mlir::ValueRange &operands) {
1795 for (mlir::Value operand : operands)
1796 if (!mlir::isa<acc::AttachOp, acc::CopyinOp, acc::CopyoutOp, acc::CreateOp,
1797 acc::DeleteOp, acc::DetachOp, acc::DevicePtrOp,
1798 acc::GetDevicePtrOp, acc::NoCreateOp, acc::PresentOp,
1799 acc::MapInfoOp>(Val: operand.getDefiningOp()))
1800 return op.emitError(
1801 "expect data entry/exit operation or acc.getdeviceptr "
1802 "as defining op");
1803 return success();
1804}
1805
1806template <typename OpT, typename RecipeOpT>
1807static LogicalResult checkPrivateOperands(mlir::Operation *accConstructOp,
1808 const mlir::ValueRange &operands,
1809 llvm::StringRef operandName) {
1810 llvm::DenseSet<Value> set;
1811 for (mlir::Value operand : operands) {
1812 if (!mlir::isa<OpT>(operand.getDefiningOp()))
1813 return accConstructOp->emitOpError()
1814 << "expected " << operandName << " as defining op";
1815 if (!set.insert(V: operand).second)
1816 return accConstructOp->emitOpError()
1817 << operandName << " operand appears more than once";
1818 }
1819 return success();
1820}
1821
1822unsigned ParallelOp::getNumDataOperands() {
1823 return getReductionOperands().size() + getPrivateOperands().size() +
1824 getFirstprivateOperands().size() + getDataClauseOperands().size();
1825}
1826
1827Value ParallelOp::getDataOperand(unsigned i) {
1828 unsigned numOptional = getAsyncOperands().size();
1829 numOptional += getNumGangs().size();
1830 numOptional += getNumWorkers().size();
1831 numOptional += getVectorLength().size();
1832 numOptional += getIfCond() ? 1 : 0;
1833 numOptional += getSelfCond() ? 1 : 0;
1834 return getOperand(i: getWaitOperands().size() + numOptional + i);
1835}
1836
1837template <typename Op>
1838static LogicalResult verifyDeviceTypeCountMatch(Op op, OperandRange operands,
1839 ArrayAttr deviceTypes,
1840 llvm::StringRef keyword) {
1841 if (!operands.empty() &&
1842 (!deviceTypes || deviceTypes.getValue().size() != operands.size()))
1843 return op.emitOpError() << keyword << " operands count must match "
1844 << keyword << " device_type count";
1845 return success();
1846}
1847
1848template <typename Op>
1849static LogicalResult verifyDeviceTypeAndSegmentCountMatch(
1850 Op op, OperandRange operands, DenseI32ArrayAttr segments,
1851 ArrayAttr deviceTypes, llvm::StringRef keyword, int32_t maxInSegment = 0) {
1852 std::size_t numOperandsInSegments = 0;
1853 std::size_t nbOfSegments = 0;
1854
1855 if (segments) {
1856 for (auto segCount : segments.asArrayRef()) {
1857 if (maxInSegment != 0 && segCount > maxInSegment)
1858 return op.emitOpError() << keyword << " expects a maximum of "
1859 << maxInSegment << " values per segment";
1860 numOperandsInSegments += segCount;
1861 ++nbOfSegments;
1862 }
1863 }
1864
1865 if ((numOperandsInSegments != operands.size()) ||
1866 (!deviceTypes && !operands.empty()))
1867 return op.emitOpError()
1868 << keyword << " operand count does not match count in segments";
1869 if (deviceTypes && deviceTypes.getValue().size() != nbOfSegments)
1870 return op.emitOpError()
1871 << keyword << " segment count does not match device_type count";
1872 return success();
1873}
1874
1875LogicalResult acc::ParallelOp::verify() {
1876 if (failed(Result: checkPrivateOperands<mlir::acc::PrivateOp,
1877 mlir::acc::PrivateRecipeOp>(
1878 accConstructOp: *this, operands: getPrivateOperands(), operandName: "private")))
1879 return failure();
1880 if (failed(Result: checkPrivateOperands<mlir::acc::FirstprivateOp,
1881 mlir::acc::FirstprivateRecipeOp>(
1882 accConstructOp: *this, operands: getFirstprivateOperands(), operandName: "firstprivate")))
1883 return failure();
1884 if (failed(Result: checkPrivateOperands<mlir::acc::ReductionOp,
1885 mlir::acc::ReductionRecipeOp>(
1886 accConstructOp: *this, operands: getReductionOperands(), operandName: "reduction")))
1887 return failure();
1888
1889 if (failed(Result: verifyDeviceTypeAndSegmentCountMatch(
1890 op: *this, operands: getNumGangs(), segments: getNumGangsSegmentsAttr(),
1891 deviceTypes: getNumGangsDeviceTypeAttr(), keyword: "num_gangs", maxInSegment: 3)))
1892 return failure();
1893
1894 if (failed(Result: verifyDeviceTypeAndSegmentCountMatch(
1895 op: *this, operands: getWaitOperands(), segments: getWaitOperandsSegmentsAttr(),
1896 deviceTypes: getWaitOperandsDeviceTypeAttr(), keyword: "wait")))
1897 return failure();
1898
1899 if (failed(Result: verifyDeviceTypeCountMatch(op: *this, operands: getNumWorkers(),
1900 deviceTypes: getNumWorkersDeviceTypeAttr(),
1901 keyword: "num_workers")))
1902 return failure();
1903
1904 if (failed(Result: verifyDeviceTypeCountMatch(op: *this, operands: getVectorLength(),
1905 deviceTypes: getVectorLengthDeviceTypeAttr(),
1906 keyword: "vector_length")))
1907 return failure();
1908
1909 if (failed(Result: verifyDeviceTypeCountMatch(op: *this, operands: getAsyncOperands(),
1910 deviceTypes: getAsyncOperandsDeviceTypeAttr(),
1911 keyword: "async")))
1912 return failure();
1913
1914 if (failed(Result: checkWaitAndAsyncConflict<acc::ParallelOp>(op: *this)))
1915 return failure();
1916
1917 return checkDataOperands<acc::ParallelOp>(op: *this, operands: getDataClauseOperands());
1918}
1919
1920static mlir::Value
1921getValueInDeviceTypeSegment(std::optional<mlir::ArrayAttr> arrayAttr,
1922 mlir::Operation::operand_range range,
1923 mlir::acc::DeviceType deviceType) {
1924 if (!arrayAttr)
1925 return {};
1926 if (auto pos = findSegment(segments: *arrayAttr, deviceType))
1927 return range[*pos];
1928 return {};
1929}
1930
1931bool acc::ParallelOp::hasAsyncOnly() {
1932 return hasAsyncOnly(deviceType: mlir::acc::DeviceType::None);
1933}
1934
1935bool acc::ParallelOp::hasAsyncOnly(mlir::acc::DeviceType deviceType) {
1936 return hasDeviceType(arrayAttr: getAsyncOnly(), deviceType);
1937}
1938
1939mlir::Value acc::ParallelOp::getAsyncValue() {
1940 return getAsyncValue(deviceType: mlir::acc::DeviceType::None);
1941}
1942
1943mlir::Value acc::ParallelOp::getAsyncValue(mlir::acc::DeviceType deviceType) {
1944 return getValueInDeviceTypeSegment(arrayAttr: getAsyncOperandsDeviceType(),
1945 range: getAsyncOperands(), deviceType);
1946}
1947
1948mlir::Value acc::ParallelOp::getNumWorkersValue() {
1949 return getNumWorkersValue(deviceType: mlir::acc::DeviceType::None);
1950}
1951
1952mlir::Value
1953acc::ParallelOp::getNumWorkersValue(mlir::acc::DeviceType deviceType) {
1954 return getValueInDeviceTypeSegment(arrayAttr: getNumWorkersDeviceType(), range: getNumWorkers(),
1955 deviceType);
1956}
1957
1958mlir::Value acc::ParallelOp::getVectorLengthValue() {
1959 return getVectorLengthValue(deviceType: mlir::acc::DeviceType::None);
1960}
1961
1962mlir::Value
1963acc::ParallelOp::getVectorLengthValue(mlir::acc::DeviceType deviceType) {
1964 return getValueInDeviceTypeSegment(arrayAttr: getVectorLengthDeviceType(),
1965 range: getVectorLength(), deviceType);
1966}
1967
1968mlir::Operation::operand_range ParallelOp::getNumGangsValues() {
1969 return getNumGangsValues(deviceType: mlir::acc::DeviceType::None);
1970}
1971
1972mlir::Operation::operand_range
1973ParallelOp::getNumGangsValues(mlir::acc::DeviceType deviceType) {
1974 return getValuesFromSegments(arrayAttr: getNumGangsDeviceType(), range: getNumGangs(),
1975 segments: getNumGangsSegments(), deviceType);
1976}
1977
1978static bool hasAnyGangWorkerVectorForDeviceType(
1979 std::optional<mlir::ArrayAttr> numGangsDeviceType,
1980 mlir::Operation::operand_range numGangs,
1981 std::optional<llvm::ArrayRef<int32_t>> numGangsSegments,
1982 std::optional<mlir::ArrayAttr> numWorkersDeviceType,
1983 mlir::Operation::operand_range numWorkers,
1984 std::optional<mlir::ArrayAttr> vectorLengthDeviceType,
1985 mlir::Operation::operand_range vectorLength,
1986 mlir::acc::DeviceType deviceType) {
1987 return !getValuesFromSegments(arrayAttr: numGangsDeviceType, range: numGangs, segments: numGangsSegments,
1988 deviceType)
1989 .empty() ||
1990 getValueInDeviceTypeSegment(arrayAttr: numWorkersDeviceType, range: numWorkers,
1991 deviceType) ||
1992 getValueInDeviceTypeSegment(arrayAttr: vectorLengthDeviceType, range: vectorLength,
1993 deviceType);
1994}
1995
1996bool acc::ParallelOp::hasAnyGangWorkerVector(mlir::acc::DeviceType deviceType) {
1997 return hasAnyGangWorkerVectorForDeviceType(
1998 numGangsDeviceType: getNumGangsDeviceType(), numGangs: getNumGangs(), numGangsSegments: getNumGangsSegments(),
1999 numWorkersDeviceType: getNumWorkersDeviceType(), numWorkers: getNumWorkers(), vectorLengthDeviceType: getVectorLengthDeviceType(),
2000 vectorLength: getVectorLength(), deviceType);
2001}
2002
2003bool acc::ParallelOp::isEffectivelySerial() {
2004 return isGangWorkerVectorAllOne(op: *this);
2005}
2006
2007bool acc::ParallelOp::hasWaitOnly() {
2008 return hasWaitOnly(deviceType: mlir::acc::DeviceType::None);
2009}
2010
2011bool acc::ParallelOp::hasWaitOnly(mlir::acc::DeviceType deviceType) {
2012 return hasDeviceType(arrayAttr: getWaitOnly(), deviceType);
2013}
2014
2015mlir::Operation::operand_range ParallelOp::getWaitValues() {
2016 return getWaitValues(deviceType: mlir::acc::DeviceType::None);
2017}
2018
2019mlir::Operation::operand_range
2020ParallelOp::getWaitValues(mlir::acc::DeviceType deviceType) {
2021 return getWaitValuesWithoutDevnum(
2022 deviceTypeAttr: getWaitOperandsDeviceType(), operands: getWaitOperands(), segments: getWaitOperandsSegments(),
2023 hasWaitDevnum: getHasWaitDevnum(), deviceType);
2024}
2025
2026mlir::Value ParallelOp::getWaitDevnum() {
2027 return getWaitDevnum(deviceType: mlir::acc::DeviceType::None);
2028}
2029
2030mlir::Value ParallelOp::getWaitDevnum(mlir::acc::DeviceType deviceType) {
2031 return getWaitDevnumValue(deviceTypeAttr: getWaitOperandsDeviceType(), operands: getWaitOperands(),
2032 segments: getWaitOperandsSegments(), hasWaitDevnum: getHasWaitDevnum(),
2033 deviceType);
2034}
2035
2036void ParallelOp::build(mlir::OpBuilder &odsBuilder,
2037 mlir::OperationState &odsState,
2038 mlir::ValueRange numGangs, mlir::ValueRange numWorkers,
2039 mlir::ValueRange vectorLength,
2040 mlir::ValueRange asyncOperands,
2041 mlir::ValueRange waitOperands, mlir::Value ifCond,
2042 mlir::Value selfCond, mlir::ValueRange reductionOperands,
2043 mlir::ValueRange gangPrivateOperands,
2044 mlir::ValueRange gangFirstPrivateOperands,
2045 mlir::ValueRange dataClauseOperands) {
2046 ParallelOp::build(
2047 odsBuilder, odsState, asyncOperands, /*asyncOperandsDeviceType=*/nullptr,
2048 /*asyncOnly=*/nullptr, waitOperands, /*waitOperandsSegments=*/nullptr,
2049 /*waitOperandsDeviceType=*/nullptr, /*hasWaitDevnum=*/nullptr,
2050 /*waitOnly=*/nullptr, numGangs, /*numGangsSegments=*/nullptr,
2051 /*numGangsDeviceType=*/nullptr, numWorkers,
2052 /*numWorkersDeviceType=*/nullptr, vectorLength,
2053 /*vectorLengthDeviceType=*/nullptr, ifCond, selfCond,
2054 /*selfAttr=*/false, reductionOperands, privateOperands: gangPrivateOperands,
2055 firstprivateOperands: gangFirstPrivateOperands, dataClauseOperands,
2056 /*defaultAttr=*/nullptr, /*combined=*/false);
2057}
2058
2059void acc::ParallelOp::addNumWorkersOperand(
2060 MLIRContext *context, mlir::Value newValue,
2061 llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
2062 setNumWorkersDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(
2063 context, existingDeviceTypes: getNumWorkersDeviceTypeAttr(), newDeviceTypes: effectiveDeviceTypes, arguments: newValue,
2064 argCollection: getNumWorkersMutable()));
2065}
2066void acc::ParallelOp::addVectorLengthOperand(
2067 MLIRContext *context, mlir::Value newValue,
2068 llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
2069 setVectorLengthDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(
2070 context, existingDeviceTypes: getVectorLengthDeviceTypeAttr(), newDeviceTypes: effectiveDeviceTypes, arguments: newValue,
2071 argCollection: getVectorLengthMutable()));
2072}
2073
2074void acc::ParallelOp::addAsyncOnly(
2075 MLIRContext *context, llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
2076 setAsyncOnlyAttr(addDeviceTypeAffectedOperandHelper(
2077 context, existingDeviceTypes: getAsyncOnlyAttr(), newDeviceTypes: effectiveDeviceTypes));
2078}
2079
2080void acc::ParallelOp::addAsyncOperand(
2081 MLIRContext *context, mlir::Value newValue,
2082 llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
2083 setAsyncOperandsDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(
2084 context, existingDeviceTypes: getAsyncOperandsDeviceTypeAttr(), newDeviceTypes: effectiveDeviceTypes, arguments: newValue,
2085 argCollection: getAsyncOperandsMutable()));
2086}
2087
2088void acc::ParallelOp::addNumGangsOperands(
2089 MLIRContext *context, mlir::ValueRange newValues,
2090 llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
2091 llvm::SmallVector<int32_t> segments;
2092 if (getNumGangsSegments())
2093 llvm::copy(Range: *getNumGangsSegments(), Out: std::back_inserter(x&: segments));
2094
2095 setNumGangsDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(
2096 context, existingDeviceTypes: getNumGangsDeviceTypeAttr(), newDeviceTypes: effectiveDeviceTypes, arguments: newValues,
2097 argCollection: getNumGangsMutable(), segments));
2098
2099 setNumGangsSegments(segments);
2100}
2101void acc::ParallelOp::addWaitOnly(
2102 MLIRContext *context, llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
2103 setWaitOnlyAttr(addDeviceTypeAffectedOperandHelper(context, existingDeviceTypes: getWaitOnlyAttr(),
2104 newDeviceTypes: effectiveDeviceTypes));
2105}
2106void acc::ParallelOp::addWaitOperands(
2107 MLIRContext *context, bool hasDevnum, mlir::ValueRange newValues,
2108 llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
2109
2110 llvm::SmallVector<int32_t> segments;
2111 if (getWaitOperandsSegments())
2112 llvm::copy(Range: *getWaitOperandsSegments(), Out: std::back_inserter(x&: segments));
2113
2114 setWaitOperandsDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(
2115 context, existingDeviceTypes: getWaitOperandsDeviceTypeAttr(), newDeviceTypes: effectiveDeviceTypes, arguments: newValues,
2116 argCollection: getWaitOperandsMutable(), segments));
2117 setWaitOperandsSegments(segments);
2118
2119 llvm::SmallVector<mlir::Attribute> hasDevnums;
2120 if (getHasWaitDevnumAttr())
2121 llvm::copy(Range: getHasWaitDevnumAttr(), Out: std::back_inserter(x&: hasDevnums));
2122 hasDevnums.insert(
2123 I: hasDevnums.end(),
2124 NumToInsert: std::max(a: effectiveDeviceTypes.size(), b: static_cast<size_t>(1)),
2125 Elt: mlir::BoolAttr::get(context, value: hasDevnum));
2126 setHasWaitDevnumAttr(mlir::ArrayAttr::get(context, value: hasDevnums));
2127}
2128
2129void acc::ParallelOp::addPrivatization(MLIRContext *context,
2130 mlir::acc::PrivateOp op,
2131 mlir::acc::PrivateRecipeOp recipe) {
2132 op.setRecipeAttr(mlir::SymbolRefAttr::get(ctx: context, value: recipe.getSymName()));
2133 getPrivateOperandsMutable().append(values: op.getResult());
2134}
2135
2136void acc::ParallelOp::addFirstPrivatization(
2137 MLIRContext *context, mlir::acc::FirstprivateOp op,
2138 mlir::acc::FirstprivateRecipeOp recipe) {
2139 op.setRecipeAttr(mlir::SymbolRefAttr::get(ctx: context, value: recipe.getSymName()));
2140 getFirstprivateOperandsMutable().append(values: op.getResult());
2141}
2142
2143void acc::ParallelOp::addReduction(MLIRContext *context,
2144 mlir::acc::ReductionOp op,
2145 mlir::acc::ReductionRecipeOp recipe) {
2146 op.setRecipeAttr(mlir::SymbolRefAttr::get(ctx: context, value: recipe.getSymName()));
2147 getReductionOperandsMutable().append(values: op.getResult());
2148}
2149
2150static ParseResult parseNumGangs(
2151 mlir::OpAsmParser &parser,
2152 llvm::SmallVectorImpl<mlir::OpAsmParser::UnresolvedOperand> &operands,
2153 llvm::SmallVectorImpl<Type> &types, mlir::ArrayAttr &deviceTypes,
2154 mlir::DenseI32ArrayAttr &segments) {
2155 llvm::SmallVector<DeviceTypeAttr> attributes;
2156 llvm::SmallVector<int32_t> seg;
2157
2158 do {
2159 if (failed(Result: parser.parseLBrace()))
2160 return failure();
2161
2162 int32_t crtOperandsSize = operands.size();
2163 if (failed(Result: parser.parseCommaSeparatedList(
2164 delimiter: mlir::AsmParser::Delimiter::None, parseElementFn: [&]() {
2165 if (parser.parseOperand(result&: operands.emplace_back()) ||
2166 parser.parseColonType(result&: types.emplace_back()))
2167 return failure();
2168 return success();
2169 })))
2170 return failure();
2171 seg.push_back(Elt: operands.size() - crtOperandsSize);
2172
2173 if (failed(Result: parser.parseRBrace()))
2174 return failure();
2175
2176 if (succeeded(Result: parser.parseOptionalLSquare())) {
2177 if (parser.parseAttribute(result&: attributes.emplace_back()) ||
2178 parser.parseRSquare())
2179 return failure();
2180 } else {
2181 attributes.push_back(Elt: mlir::acc::DeviceTypeAttr::get(
2182 context: parser.getContext(), value: mlir::acc::DeviceType::None));
2183 }
2184 } while (succeeded(Result: parser.parseOptionalComma()));
2185
2186 llvm::SmallVector<mlir::Attribute> arrayAttr(attributes.begin(),
2187 attributes.end());
2188 deviceTypes = ArrayAttr::get(context: parser.getContext(), value: arrayAttr);
2189 segments = DenseI32ArrayAttr::get(context: parser.getContext(), content: seg);
2190
2191 return success();
2192}
2193
2194static void printSingleDeviceType(mlir::OpAsmPrinter &p, mlir::Attribute attr) {
2195 auto deviceTypeAttr = mlir::dyn_cast<mlir::acc::DeviceTypeAttr>(Val&: attr);
2196 if (deviceTypeAttr.getValue() != mlir::acc::DeviceType::None)
2197 p << " [" << attr << "]";
2198}
2199
2200static void printNumGangs(mlir::OpAsmPrinter &p, mlir::Operation *op,
2201 mlir::OperandRange operands, mlir::TypeRange types,
2202 std::optional<mlir::ArrayAttr> deviceTypes,
2203 std::optional<mlir::DenseI32ArrayAttr> segments) {
2204 unsigned opIdx = 0;
2205 llvm::interleaveComma(c: llvm::enumerate(First&: *deviceTypes), os&: p, each_fn: [&](auto it) {
2206 p << "{";
2207 llvm::interleaveComma(
2208 llvm::seq<int32_t>(0, (*segments)[it.index()]), p, [&](auto it) {
2209 p << operands[opIdx] << " : " << operands[opIdx].getType();
2210 ++opIdx;
2211 });
2212 p << "}";
2213 printSingleDeviceType(p, it.value());
2214 });
2215}
2216
2217static ParseResult parseDeviceTypeOperandsWithSegment(
2218 mlir::OpAsmParser &parser,
2219 llvm::SmallVectorImpl<mlir::OpAsmParser::UnresolvedOperand> &operands,
2220 llvm::SmallVectorImpl<Type> &types, mlir::ArrayAttr &deviceTypes,
2221 mlir::DenseI32ArrayAttr &segments) {
2222 llvm::SmallVector<DeviceTypeAttr> attributes;
2223 llvm::SmallVector<int32_t> seg;
2224
2225 do {
2226 if (failed(Result: parser.parseLBrace()))
2227 return failure();
2228
2229 int32_t crtOperandsSize = operands.size();
2230
2231 if (failed(Result: parser.parseCommaSeparatedList(
2232 delimiter: mlir::AsmParser::Delimiter::None, parseElementFn: [&]() {
2233 if (parser.parseOperand(result&: operands.emplace_back()) ||
2234 parser.parseColonType(result&: types.emplace_back()))
2235 return failure();
2236 return success();
2237 })))
2238 return failure();
2239
2240 seg.push_back(Elt: operands.size() - crtOperandsSize);
2241
2242 if (failed(Result: parser.parseRBrace()))
2243 return failure();
2244
2245 if (succeeded(Result: parser.parseOptionalLSquare())) {
2246 if (parser.parseAttribute(result&: attributes.emplace_back()) ||
2247 parser.parseRSquare())
2248 return failure();
2249 } else {
2250 attributes.push_back(Elt: mlir::acc::DeviceTypeAttr::get(
2251 context: parser.getContext(), value: mlir::acc::DeviceType::None));
2252 }
2253 } while (succeeded(Result: parser.parseOptionalComma()));
2254
2255 llvm::SmallVector<mlir::Attribute> arrayAttr(attributes.begin(),
2256 attributes.end());
2257 deviceTypes = ArrayAttr::get(context: parser.getContext(), value: arrayAttr);
2258 segments = DenseI32ArrayAttr::get(context: parser.getContext(), content: seg);
2259
2260 return success();
2261}
2262
2263static void printDeviceTypeOperandsWithSegment(
2264 mlir::OpAsmPrinter &p, mlir::Operation *op, mlir::OperandRange operands,
2265 mlir::TypeRange types, std::optional<mlir::ArrayAttr> deviceTypes,
2266 std::optional<mlir::DenseI32ArrayAttr> segments) {
2267 unsigned opIdx = 0;
2268 llvm::interleaveComma(c: llvm::enumerate(First&: *deviceTypes), os&: p, each_fn: [&](auto it) {
2269 p << "{";
2270 llvm::interleaveComma(
2271 llvm::seq<int32_t>(0, (*segments)[it.index()]), p, [&](auto it) {
2272 p << operands[opIdx] << " : " << operands[opIdx].getType();
2273 ++opIdx;
2274 });
2275 p << "}";
2276 printSingleDeviceType(p, it.value());
2277 });
2278}
2279
2280static ParseResult parseWaitClause(
2281 mlir::OpAsmParser &parser,
2282 llvm::SmallVectorImpl<mlir::OpAsmParser::UnresolvedOperand> &operands,
2283 llvm::SmallVectorImpl<Type> &types, mlir::ArrayAttr &deviceTypes,
2284 mlir::DenseI32ArrayAttr &segments, mlir::ArrayAttr &hasDevNum,
2285 mlir::ArrayAttr &keywordOnly) {
2286 llvm::SmallVector<mlir::Attribute> deviceTypeAttrs, keywordAttrs, devnum;
2287 llvm::SmallVector<int32_t> seg;
2288
2289 bool needCommaBeforeOperands = false;
2290
2291 // Keyword only
2292 if (failed(Result: parser.parseOptionalLParen())) {
2293 keywordAttrs.push_back(Elt: mlir::acc::DeviceTypeAttr::get(
2294 context: parser.getContext(), value: mlir::acc::DeviceType::None));
2295 keywordOnly = ArrayAttr::get(context: parser.getContext(), value: keywordAttrs);
2296 return success();
2297 }
2298
2299 // Parse keyword only attributes
2300 if (succeeded(Result: parser.parseOptionalLSquare())) {
2301 if (failed(Result: parser.parseCommaSeparatedList(parseElementFn: [&]() {
2302 if (parser.parseAttribute(result&: keywordAttrs.emplace_back()))
2303 return failure();
2304 return success();
2305 })))
2306 return failure();
2307 if (parser.parseRSquare())
2308 return failure();
2309 needCommaBeforeOperands = true;
2310 }
2311
2312 if (needCommaBeforeOperands && failed(Result: parser.parseComma()))
2313 return failure();
2314
2315 do {
2316 if (failed(Result: parser.parseLBrace()))
2317 return failure();
2318
2319 int32_t crtOperandsSize = operands.size();
2320
2321 if (succeeded(Result: parser.parseOptionalKeyword(keyword: "devnum"))) {
2322 if (failed(Result: parser.parseColon()))
2323 return failure();
2324 devnum.push_back(Elt: BoolAttr::get(context: parser.getContext(), value: true));
2325 } else {
2326 devnum.push_back(Elt: BoolAttr::get(context: parser.getContext(), value: false));
2327 }
2328
2329 if (failed(Result: parser.parseCommaSeparatedList(
2330 delimiter: mlir::AsmParser::Delimiter::None, parseElementFn: [&]() {
2331 if (parser.parseOperand(result&: operands.emplace_back()) ||
2332 parser.parseColonType(result&: types.emplace_back()))
2333 return failure();
2334 return success();
2335 })))
2336 return failure();
2337
2338 seg.push_back(Elt: operands.size() - crtOperandsSize);
2339
2340 if (failed(Result: parser.parseRBrace()))
2341 return failure();
2342
2343 if (succeeded(Result: parser.parseOptionalLSquare())) {
2344 if (parser.parseAttribute(result&: deviceTypeAttrs.emplace_back()) ||
2345 parser.parseRSquare())
2346 return failure();
2347 } else {
2348 deviceTypeAttrs.push_back(Elt: mlir::acc::DeviceTypeAttr::get(
2349 context: parser.getContext(), value: mlir::acc::DeviceType::None));
2350 }
2351 } while (succeeded(Result: parser.parseOptionalComma()));
2352
2353 if (failed(Result: parser.parseRParen()))
2354 return failure();
2355
2356 deviceTypes = ArrayAttr::get(context: parser.getContext(), value: deviceTypeAttrs);
2357 keywordOnly = ArrayAttr::get(context: parser.getContext(), value: keywordAttrs);
2358 segments = DenseI32ArrayAttr::get(context: parser.getContext(), content: seg);
2359 hasDevNum = ArrayAttr::get(context: parser.getContext(), value: devnum);
2360
2361 return success();
2362}
2363
2364static bool hasOnlyDeviceTypeNone(std::optional<mlir::ArrayAttr> attrs) {
2365 if (!hasDeviceTypeValues(arrayAttr: attrs))
2366 return false;
2367 if (attrs->size() != 1)
2368 return false;
2369 if (auto deviceTypeAttr =
2370 mlir::dyn_cast<mlir::acc::DeviceTypeAttr>(Val: (*attrs)[0]))
2371 return deviceTypeAttr.getValue() == mlir::acc::DeviceType::None;
2372 return false;
2373}
2374
2375static void printWaitClause(mlir::OpAsmPrinter &p, mlir::Operation *op,
2376 mlir::OperandRange operands, mlir::TypeRange types,
2377 std::optional<mlir::ArrayAttr> deviceTypes,
2378 std::optional<mlir::DenseI32ArrayAttr> segments,
2379 std::optional<mlir::ArrayAttr> hasDevNum,
2380 std::optional<mlir::ArrayAttr> keywordOnly) {
2381
2382 if (operands.begin() == operands.end() && hasOnlyDeviceTypeNone(attrs: keywordOnly))
2383 return;
2384
2385 p << "(";
2386
2387 printDeviceTypes(p, deviceTypes: keywordOnly);
2388 if (hasDeviceTypeValues(arrayAttr: keywordOnly) && hasDeviceTypeValues(arrayAttr: deviceTypes))
2389 p << ", ";
2390
2391 if (hasDeviceTypeValues(arrayAttr: deviceTypes)) {
2392 unsigned opIdx = 0;
2393 llvm::interleaveComma(c: llvm::enumerate(First&: *deviceTypes), os&: p, each_fn: [&](auto it) {
2394 p << "{";
2395 auto boolAttr = mlir::dyn_cast<mlir::BoolAttr>((*hasDevNum)[it.index()]);
2396 if (boolAttr && boolAttr.getValue())
2397 p << "devnum: ";
2398 llvm::interleaveComma(
2399 llvm::seq<int32_t>(0, (*segments)[it.index()]), p, [&](auto it) {
2400 p << operands[opIdx] << " : " << operands[opIdx].getType();
2401 ++opIdx;
2402 });
2403 p << "}";
2404 printSingleDeviceType(p, it.value());
2405 });
2406 }
2407
2408 p << ")";
2409}
2410
2411static ParseResult parseDeviceTypeOperands(
2412 mlir::OpAsmParser &parser,
2413 llvm::SmallVectorImpl<mlir::OpAsmParser::UnresolvedOperand> &operands,
2414 llvm::SmallVectorImpl<Type> &types, mlir::ArrayAttr &deviceTypes) {
2415 llvm::SmallVector<DeviceTypeAttr> attributes;
2416 if (failed(Result: parser.parseCommaSeparatedList(parseElementFn: [&]() {
2417 if (parser.parseOperand(result&: operands.emplace_back()) ||
2418 parser.parseColonType(result&: types.emplace_back()))
2419 return failure();
2420 if (succeeded(Result: parser.parseOptionalLSquare())) {
2421 if (parser.parseAttribute(result&: attributes.emplace_back()) ||
2422 parser.parseRSquare())
2423 return failure();
2424 } else {
2425 attributes.push_back(Elt: mlir::acc::DeviceTypeAttr::get(
2426 context: parser.getContext(), value: mlir::acc::DeviceType::None));
2427 }
2428 return success();
2429 })))
2430 return failure();
2431 llvm::SmallVector<mlir::Attribute> arrayAttr(attributes.begin(),
2432 attributes.end());
2433 deviceTypes = ArrayAttr::get(context: parser.getContext(), value: arrayAttr);
2434 return success();
2435}
2436
2437static void
2438printDeviceTypeOperands(mlir::OpAsmPrinter &p, mlir::Operation *op,
2439 mlir::OperandRange operands, mlir::TypeRange types,
2440 std::optional<mlir::ArrayAttr> deviceTypes) {
2441 if (!hasDeviceTypeValues(arrayAttr: deviceTypes))
2442 return;
2443 llvm::interleaveComma(c: llvm::zip(t&: *deviceTypes, u&: operands), os&: p, each_fn: [&](auto it) {
2444 p << std::get<1>(it) << " : " << std::get<1>(it).getType();
2445 printSingleDeviceType(p, std::get<0>(it));
2446 });
2447}
2448
2449static ParseResult parseDeviceTypeOperandsWithKeywordOnly(
2450 mlir::OpAsmParser &parser,
2451 llvm::SmallVectorImpl<mlir::OpAsmParser::UnresolvedOperand> &operands,
2452 llvm::SmallVectorImpl<Type> &types, mlir::ArrayAttr &deviceTypes,
2453 mlir::ArrayAttr &keywordOnlyDeviceType) {
2454
2455 llvm::SmallVector<mlir::Attribute> keywordOnlyDeviceTypeAttributes;
2456 bool needCommaBeforeOperands = false;
2457
2458 if (failed(Result: parser.parseOptionalLParen())) {
2459 // Keyword only
2460 keywordOnlyDeviceTypeAttributes.push_back(Elt: mlir::acc::DeviceTypeAttr::get(
2461 context: parser.getContext(), value: mlir::acc::DeviceType::None));
2462 keywordOnlyDeviceType =
2463 ArrayAttr::get(context: parser.getContext(), value: keywordOnlyDeviceTypeAttributes);
2464 return success();
2465 }
2466
2467 // Parse keyword only attributes
2468 if (succeeded(Result: parser.parseOptionalLSquare())) {
2469 // Parse keyword only attributes
2470 if (failed(Result: parser.parseCommaSeparatedList(parseElementFn: [&]() {
2471 if (parser.parseAttribute(
2472 result&: keywordOnlyDeviceTypeAttributes.emplace_back()))
2473 return failure();
2474 return success();
2475 })))
2476 return failure();
2477 if (parser.parseRSquare())
2478 return failure();
2479 keywordOnlyDeviceType =
2480 ArrayAttr::get(context: parser.getContext(), value: keywordOnlyDeviceTypeAttributes);
2481 needCommaBeforeOperands = true;
2482 }
2483
2484 if (needCommaBeforeOperands) {
2485 if (succeeded(Result: parser.parseOptionalRParen()))
2486 return success();
2487 if (failed(Result: parser.parseComma()))
2488 return failure();
2489 }
2490
2491 llvm::SmallVector<DeviceTypeAttr> attributes;
2492 if (failed(Result: parser.parseCommaSeparatedList(parseElementFn: [&]() {
2493 if (parser.parseOperand(result&: operands.emplace_back()) ||
2494 parser.parseColonType(result&: types.emplace_back()))
2495 return failure();
2496 if (succeeded(Result: parser.parseOptionalLSquare())) {
2497 if (parser.parseAttribute(result&: attributes.emplace_back()) ||
2498 parser.parseRSquare())
2499 return failure();
2500 } else {
2501 attributes.push_back(Elt: mlir::acc::DeviceTypeAttr::get(
2502 context: parser.getContext(), value: mlir::acc::DeviceType::None));
2503 }
2504 return success();
2505 })))
2506 return failure();
2507
2508 if (failed(Result: parser.parseRParen()))
2509 return failure();
2510
2511 llvm::SmallVector<mlir::Attribute> arrayAttr(attributes.begin(),
2512 attributes.end());
2513 deviceTypes = ArrayAttr::get(context: parser.getContext(), value: arrayAttr);
2514 return success();
2515}
2516
2517static void printDeviceTypeOperandsWithKeywordOnly(
2518 mlir::OpAsmPrinter &p, mlir::Operation *op, mlir::OperandRange operands,
2519 mlir::TypeRange types, std::optional<mlir::ArrayAttr> deviceTypes,
2520 std::optional<mlir::ArrayAttr> keywordOnlyDeviceTypes) {
2521
2522 if (operands.begin() == operands.end() &&
2523 hasOnlyDeviceTypeNone(attrs: keywordOnlyDeviceTypes)) {
2524 return;
2525 }
2526
2527 p << "(";
2528 printDeviceTypes(p, deviceTypes: keywordOnlyDeviceTypes);
2529 if (hasDeviceTypeValues(arrayAttr: keywordOnlyDeviceTypes) &&
2530 hasDeviceTypeValues(arrayAttr: deviceTypes))
2531 p << ", ";
2532 printDeviceTypeOperands(p, op, operands, types, deviceTypes);
2533 p << ")";
2534}
2535
2536static ParseResult parseOperandWithKeywordOnly(
2537 mlir::OpAsmParser &parser,
2538 std::optional<OpAsmParser::UnresolvedOperand> &operand,
2539 mlir::Type &operandType, mlir::UnitAttr &attr) {
2540 // Keyword only
2541 if (failed(Result: parser.parseOptionalLParen())) {
2542 attr = mlir::UnitAttr::get(context: parser.getContext());
2543 return success();
2544 }
2545
2546 OpAsmParser::UnresolvedOperand op;
2547 if (failed(Result: parser.parseOperand(result&: op)))
2548 return failure();
2549 operand = op;
2550 if (failed(Result: parser.parseColon()))
2551 return failure();
2552 if (failed(Result: parser.parseType(result&: operandType)))
2553 return failure();
2554 if (failed(Result: parser.parseRParen()))
2555 return failure();
2556
2557 return success();
2558}
2559
2560static void printOperandWithKeywordOnly(mlir::OpAsmPrinter &p,
2561 mlir::Operation *op,
2562 std::optional<mlir::Value> operand,
2563 mlir::Type operandType,
2564 mlir::UnitAttr attr) {
2565 if (attr)
2566 return;
2567
2568 p << "(";
2569 p.printOperand(value: *operand);
2570 p << " : ";
2571 p.printType(type: operandType);
2572 p << ")";
2573}
2574
2575static ParseResult parseOperandsWithKeywordOnly(
2576 mlir::OpAsmParser &parser,
2577 llvm::SmallVectorImpl<mlir::OpAsmParser::UnresolvedOperand> &operands,
2578 llvm::SmallVectorImpl<Type> &types, mlir::UnitAttr &attr) {
2579 // Keyword only
2580 if (failed(Result: parser.parseOptionalLParen())) {
2581 attr = mlir::UnitAttr::get(context: parser.getContext());
2582 return success();
2583 }
2584
2585 if (failed(Result: parser.parseCommaSeparatedList(parseElementFn: [&]() {
2586 if (parser.parseOperand(result&: operands.emplace_back()))
2587 return failure();
2588 return success();
2589 })))
2590 return failure();
2591 if (failed(Result: parser.parseColon()))
2592 return failure();
2593 if (failed(Result: parser.parseCommaSeparatedList(parseElementFn: [&]() {
2594 if (parser.parseType(result&: types.emplace_back()))
2595 return failure();
2596 return success();
2597 })))
2598 return failure();
2599 if (failed(Result: parser.parseRParen()))
2600 return failure();
2601
2602 return success();
2603}
2604
2605static void printOperandsWithKeywordOnly(mlir::OpAsmPrinter &p,
2606 mlir::Operation *op,
2607 mlir::OperandRange operands,
2608 mlir::TypeRange types,
2609 mlir::UnitAttr attr) {
2610 if (attr)
2611 return;
2612
2613 p << "(";
2614 llvm::interleaveComma(c: operands, os&: p, each_fn: [&](auto it) { p << it; });
2615 p << " : ";
2616 llvm::interleaveComma(c: types, os&: p, each_fn: [&](auto it) { p << it; });
2617 p << ")";
2618}
2619
2620static ParseResult
2621parseCombinedConstructsLoop(mlir::OpAsmParser &parser,
2622 mlir::acc::CombinedConstructsTypeAttr &attr) {
2623 if (succeeded(Result: parser.parseOptionalKeyword(keyword: "kernels"))) {
2624 attr = mlir::acc::CombinedConstructsTypeAttr::get(
2625 context: parser.getContext(), value: mlir::acc::CombinedConstructsType::KernelsLoop);
2626 } else if (succeeded(Result: parser.parseOptionalKeyword(keyword: "parallel"))) {
2627 attr = mlir::acc::CombinedConstructsTypeAttr::get(
2628 context: parser.getContext(), value: mlir::acc::CombinedConstructsType::ParallelLoop);
2629 } else if (succeeded(Result: parser.parseOptionalKeyword(keyword: "serial"))) {
2630 attr = mlir::acc::CombinedConstructsTypeAttr::get(
2631 context: parser.getContext(), value: mlir::acc::CombinedConstructsType::SerialLoop);
2632 } else {
2633 parser.emitError(loc: parser.getCurrentLocation(),
2634 message: "expected compute construct name");
2635 return failure();
2636 }
2637 return success();
2638}
2639
2640static void
2641printCombinedConstructsLoop(mlir::OpAsmPrinter &p, mlir::Operation *op,
2642 mlir::acc::CombinedConstructsTypeAttr attr) {
2643 if (attr) {
2644 switch (attr.getValue()) {
2645 case mlir::acc::CombinedConstructsType::KernelsLoop:
2646 p << "kernels";
2647 break;
2648 case mlir::acc::CombinedConstructsType::ParallelLoop:
2649 p << "parallel";
2650 break;
2651 case mlir::acc::CombinedConstructsType::SerialLoop:
2652 p << "serial";
2653 break;
2654 };
2655 }
2656}
2657
2658//===----------------------------------------------------------------------===//
2659// SerialOp
2660//===----------------------------------------------------------------------===//
2661
2662unsigned SerialOp::getNumDataOperands() {
2663 return getReductionOperands().size() + getPrivateOperands().size() +
2664 getFirstprivateOperands().size() + getDataClauseOperands().size();
2665}
2666
2667Value SerialOp::getDataOperand(unsigned i) {
2668 unsigned numOptional = getAsyncOperands().size();
2669 numOptional += getIfCond() ? 1 : 0;
2670 numOptional += getSelfCond() ? 1 : 0;
2671 return getOperand(i: getWaitOperands().size() + numOptional + i);
2672}
2673
2674bool acc::SerialOp::hasAsyncOnly() {
2675 return hasAsyncOnly(deviceType: mlir::acc::DeviceType::None);
2676}
2677
2678bool acc::SerialOp::hasAsyncOnly(mlir::acc::DeviceType deviceType) {
2679 return hasDeviceType(arrayAttr: getAsyncOnly(), deviceType);
2680}
2681
2682mlir::Value acc::SerialOp::getAsyncValue() {
2683 return getAsyncValue(deviceType: mlir::acc::DeviceType::None);
2684}
2685
2686mlir::Value acc::SerialOp::getAsyncValue(mlir::acc::DeviceType deviceType) {
2687 return getValueInDeviceTypeSegment(arrayAttr: getAsyncOperandsDeviceType(),
2688 range: getAsyncOperands(), deviceType);
2689}
2690
2691bool acc::SerialOp::hasWaitOnly() {
2692 return hasWaitOnly(deviceType: mlir::acc::DeviceType::None);
2693}
2694
2695bool acc::SerialOp::hasWaitOnly(mlir::acc::DeviceType deviceType) {
2696 return hasDeviceType(arrayAttr: getWaitOnly(), deviceType);
2697}
2698
2699mlir::Operation::operand_range SerialOp::getWaitValues() {
2700 return getWaitValues(deviceType: mlir::acc::DeviceType::None);
2701}
2702
2703mlir::Operation::operand_range
2704SerialOp::getWaitValues(mlir::acc::DeviceType deviceType) {
2705 return getWaitValuesWithoutDevnum(
2706 deviceTypeAttr: getWaitOperandsDeviceType(), operands: getWaitOperands(), segments: getWaitOperandsSegments(),
2707 hasWaitDevnum: getHasWaitDevnum(), deviceType);
2708}
2709
2710mlir::Value SerialOp::getWaitDevnum() {
2711 return getWaitDevnum(deviceType: mlir::acc::DeviceType::None);
2712}
2713
2714mlir::Value SerialOp::getWaitDevnum(mlir::acc::DeviceType deviceType) {
2715 return getWaitDevnumValue(deviceTypeAttr: getWaitOperandsDeviceType(), operands: getWaitOperands(),
2716 segments: getWaitOperandsSegments(), hasWaitDevnum: getHasWaitDevnum(),
2717 deviceType);
2718}
2719
2720LogicalResult acc::SerialOp::verify() {
2721 if (failed(Result: checkPrivateOperands<mlir::acc::PrivateOp,
2722 mlir::acc::PrivateRecipeOp>(
2723 accConstructOp: *this, operands: getPrivateOperands(), operandName: "private")))
2724 return failure();
2725 if (failed(Result: checkPrivateOperands<mlir::acc::FirstprivateOp,
2726 mlir::acc::FirstprivateRecipeOp>(
2727 accConstructOp: *this, operands: getFirstprivateOperands(), operandName: "firstprivate")))
2728 return failure();
2729 if (failed(Result: checkPrivateOperands<mlir::acc::ReductionOp,
2730 mlir::acc::ReductionRecipeOp>(
2731 accConstructOp: *this, operands: getReductionOperands(), operandName: "reduction")))
2732 return failure();
2733
2734 if (failed(Result: verifyDeviceTypeAndSegmentCountMatch(
2735 op: *this, operands: getWaitOperands(), segments: getWaitOperandsSegmentsAttr(),
2736 deviceTypes: getWaitOperandsDeviceTypeAttr(), keyword: "wait")))
2737 return failure();
2738
2739 if (failed(Result: verifyDeviceTypeCountMatch(op: *this, operands: getAsyncOperands(),
2740 deviceTypes: getAsyncOperandsDeviceTypeAttr(),
2741 keyword: "async")))
2742 return failure();
2743
2744 if (failed(Result: checkWaitAndAsyncConflict<acc::SerialOp>(op: *this)))
2745 return failure();
2746
2747 return checkDataOperands<acc::SerialOp>(op: *this, operands: getDataClauseOperands());
2748}
2749
2750void acc::SerialOp::addAsyncOnly(
2751 MLIRContext *context, llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
2752 setAsyncOnlyAttr(addDeviceTypeAffectedOperandHelper(
2753 context, existingDeviceTypes: getAsyncOnlyAttr(), newDeviceTypes: effectiveDeviceTypes));
2754}
2755
2756void acc::SerialOp::addAsyncOperand(
2757 MLIRContext *context, mlir::Value newValue,
2758 llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
2759 setAsyncOperandsDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(
2760 context, existingDeviceTypes: getAsyncOperandsDeviceTypeAttr(), newDeviceTypes: effectiveDeviceTypes, arguments: newValue,
2761 argCollection: getAsyncOperandsMutable()));
2762}
2763
2764void acc::SerialOp::addWaitOnly(
2765 MLIRContext *context, llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
2766 setWaitOnlyAttr(addDeviceTypeAffectedOperandHelper(context, existingDeviceTypes: getWaitOnlyAttr(),
2767 newDeviceTypes: effectiveDeviceTypes));
2768}
2769void acc::SerialOp::addWaitOperands(
2770 MLIRContext *context, bool hasDevnum, mlir::ValueRange newValues,
2771 llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
2772
2773 llvm::SmallVector<int32_t> segments;
2774 if (getWaitOperandsSegments())
2775 llvm::copy(Range: *getWaitOperandsSegments(), Out: std::back_inserter(x&: segments));
2776
2777 setWaitOperandsDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(
2778 context, existingDeviceTypes: getWaitOperandsDeviceTypeAttr(), newDeviceTypes: effectiveDeviceTypes, arguments: newValues,
2779 argCollection: getWaitOperandsMutable(), segments));
2780 setWaitOperandsSegments(segments);
2781
2782 llvm::SmallVector<mlir::Attribute> hasDevnums;
2783 if (getHasWaitDevnumAttr())
2784 llvm::copy(Range: getHasWaitDevnumAttr(), Out: std::back_inserter(x&: hasDevnums));
2785 hasDevnums.insert(
2786 I: hasDevnums.end(),
2787 NumToInsert: std::max(a: effectiveDeviceTypes.size(), b: static_cast<size_t>(1)),
2788 Elt: mlir::BoolAttr::get(context, value: hasDevnum));
2789 setHasWaitDevnumAttr(mlir::ArrayAttr::get(context, value: hasDevnums));
2790}
2791
2792void acc::SerialOp::addPrivatization(MLIRContext *context,
2793 mlir::acc::PrivateOp op,
2794 mlir::acc::PrivateRecipeOp recipe) {
2795 op.setRecipeAttr(mlir::SymbolRefAttr::get(ctx: context, value: recipe.getSymName()));
2796 getPrivateOperandsMutable().append(values: op.getResult());
2797}
2798
2799void acc::SerialOp::addFirstPrivatization(
2800 MLIRContext *context, mlir::acc::FirstprivateOp op,
2801 mlir::acc::FirstprivateRecipeOp recipe) {
2802 op.setRecipeAttr(mlir::SymbolRefAttr::get(ctx: context, value: recipe.getSymName()));
2803 getFirstprivateOperandsMutable().append(values: op.getResult());
2804}
2805
2806void acc::SerialOp::addReduction(MLIRContext *context,
2807 mlir::acc::ReductionOp op,
2808 mlir::acc::ReductionRecipeOp recipe) {
2809 op.setRecipeAttr(mlir::SymbolRefAttr::get(ctx: context, value: recipe.getSymName()));
2810 getReductionOperandsMutable().append(values: op.getResult());
2811}
2812
2813//===----------------------------------------------------------------------===//
2814// KernelsOp
2815//===----------------------------------------------------------------------===//
2816
2817unsigned KernelsOp::getNumDataOperands() {
2818 return getDataClauseOperands().size();
2819}
2820
2821Value KernelsOp::getDataOperand(unsigned i) {
2822 unsigned numOptional = getAsyncOperands().size();
2823 numOptional += getWaitOperands().size();
2824 numOptional += getNumGangs().size();
2825 numOptional += getNumWorkers().size();
2826 numOptional += getVectorLength().size();
2827 numOptional += getIfCond() ? 1 : 0;
2828 numOptional += getSelfCond() ? 1 : 0;
2829 return getOperand(i: numOptional + i);
2830}
2831
2832bool acc::KernelsOp::hasAsyncOnly() {
2833 return hasAsyncOnly(deviceType: mlir::acc::DeviceType::None);
2834}
2835
2836bool acc::KernelsOp::hasAsyncOnly(mlir::acc::DeviceType deviceType) {
2837 return hasDeviceType(arrayAttr: getAsyncOnly(), deviceType);
2838}
2839
2840mlir::Value acc::KernelsOp::getAsyncValue() {
2841 return getAsyncValue(deviceType: mlir::acc::DeviceType::None);
2842}
2843
2844mlir::Value acc::KernelsOp::getAsyncValue(mlir::acc::DeviceType deviceType) {
2845 return getValueInDeviceTypeSegment(arrayAttr: getAsyncOperandsDeviceType(),
2846 range: getAsyncOperands(), deviceType);
2847}
2848
2849mlir::Value acc::KernelsOp::getNumWorkersValue() {
2850 return getNumWorkersValue(deviceType: mlir::acc::DeviceType::None);
2851}
2852
2853mlir::Value
2854acc::KernelsOp::getNumWorkersValue(mlir::acc::DeviceType deviceType) {
2855 return getValueInDeviceTypeSegment(arrayAttr: getNumWorkersDeviceType(), range: getNumWorkers(),
2856 deviceType);
2857}
2858
2859mlir::Value acc::KernelsOp::getVectorLengthValue() {
2860 return getVectorLengthValue(deviceType: mlir::acc::DeviceType::None);
2861}
2862
2863mlir::Value
2864acc::KernelsOp::getVectorLengthValue(mlir::acc::DeviceType deviceType) {
2865 return getValueInDeviceTypeSegment(arrayAttr: getVectorLengthDeviceType(),
2866 range: getVectorLength(), deviceType);
2867}
2868
2869mlir::Operation::operand_range KernelsOp::getNumGangsValues() {
2870 return getNumGangsValues(deviceType: mlir::acc::DeviceType::None);
2871}
2872
2873mlir::Operation::operand_range
2874KernelsOp::getNumGangsValues(mlir::acc::DeviceType deviceType) {
2875 return getValuesFromSegments(arrayAttr: getNumGangsDeviceType(), range: getNumGangs(),
2876 segments: getNumGangsSegments(), deviceType);
2877}
2878
2879bool acc::KernelsOp::hasAnyGangWorkerVector(mlir::acc::DeviceType deviceType) {
2880 return hasAnyGangWorkerVectorForDeviceType(
2881 numGangsDeviceType: getNumGangsDeviceType(), numGangs: getNumGangs(), numGangsSegments: getNumGangsSegments(),
2882 numWorkersDeviceType: getNumWorkersDeviceType(), numWorkers: getNumWorkers(), vectorLengthDeviceType: getVectorLengthDeviceType(),
2883 vectorLength: getVectorLength(), deviceType);
2884}
2885
2886bool acc::KernelsOp::isEffectivelySerial() {
2887 return isGangWorkerVectorAllOne(op: *this);
2888}
2889
2890bool acc::KernelsOp::hasWaitOnly() {
2891 return hasWaitOnly(deviceType: mlir::acc::DeviceType::None);
2892}
2893
2894bool acc::KernelsOp::hasWaitOnly(mlir::acc::DeviceType deviceType) {
2895 return hasDeviceType(arrayAttr: getWaitOnly(), deviceType);
2896}
2897
2898mlir::Operation::operand_range KernelsOp::getWaitValues() {
2899 return getWaitValues(deviceType: mlir::acc::DeviceType::None);
2900}
2901
2902mlir::Operation::operand_range
2903KernelsOp::getWaitValues(mlir::acc::DeviceType deviceType) {
2904 return getWaitValuesWithoutDevnum(
2905 deviceTypeAttr: getWaitOperandsDeviceType(), operands: getWaitOperands(), segments: getWaitOperandsSegments(),
2906 hasWaitDevnum: getHasWaitDevnum(), deviceType);
2907}
2908
2909mlir::Value KernelsOp::getWaitDevnum() {
2910 return getWaitDevnum(deviceType: mlir::acc::DeviceType::None);
2911}
2912
2913mlir::Value KernelsOp::getWaitDevnum(mlir::acc::DeviceType deviceType) {
2914 return getWaitDevnumValue(deviceTypeAttr: getWaitOperandsDeviceType(), operands: getWaitOperands(),
2915 segments: getWaitOperandsSegments(), hasWaitDevnum: getHasWaitDevnum(),
2916 deviceType);
2917}
2918
2919LogicalResult acc::KernelsOp::verify() {
2920 if (failed(Result: verifyDeviceTypeAndSegmentCountMatch(
2921 op: *this, operands: getNumGangs(), segments: getNumGangsSegmentsAttr(),
2922 deviceTypes: getNumGangsDeviceTypeAttr(), keyword: "num_gangs", maxInSegment: 3)))
2923 return failure();
2924
2925 if (failed(Result: verifyDeviceTypeAndSegmentCountMatch(
2926 op: *this, operands: getWaitOperands(), segments: getWaitOperandsSegmentsAttr(),
2927 deviceTypes: getWaitOperandsDeviceTypeAttr(), keyword: "wait")))
2928 return failure();
2929
2930 if (failed(Result: verifyDeviceTypeCountMatch(op: *this, operands: getNumWorkers(),
2931 deviceTypes: getNumWorkersDeviceTypeAttr(),
2932 keyword: "num_workers")))
2933 return failure();
2934
2935 if (failed(Result: verifyDeviceTypeCountMatch(op: *this, operands: getVectorLength(),
2936 deviceTypes: getVectorLengthDeviceTypeAttr(),
2937 keyword: "vector_length")))
2938 return failure();
2939
2940 if (failed(Result: verifyDeviceTypeCountMatch(op: *this, operands: getAsyncOperands(),
2941 deviceTypes: getAsyncOperandsDeviceTypeAttr(),
2942 keyword: "async")))
2943 return failure();
2944
2945 if (failed(Result: checkWaitAndAsyncConflict<acc::KernelsOp>(op: *this)))
2946 return failure();
2947
2948 return checkDataOperands<acc::KernelsOp>(op: *this, operands: getDataClauseOperands());
2949}
2950
2951void acc::KernelsOp::addPrivatization(MLIRContext *context,
2952 mlir::acc::PrivateOp op,
2953 mlir::acc::PrivateRecipeOp recipe) {
2954 op.setRecipeAttr(mlir::SymbolRefAttr::get(ctx: context, value: recipe.getSymName()));
2955 getPrivateOperandsMutable().append(values: op.getResult());
2956}
2957
2958void acc::KernelsOp::addFirstPrivatization(
2959 MLIRContext *context, mlir::acc::FirstprivateOp op,
2960 mlir::acc::FirstprivateRecipeOp recipe) {
2961 op.setRecipeAttr(mlir::SymbolRefAttr::get(ctx: context, value: recipe.getSymName()));
2962 getFirstprivateOperandsMutable().append(values: op.getResult());
2963}
2964
2965void acc::KernelsOp::addReduction(MLIRContext *context,
2966 mlir::acc::ReductionOp op,
2967 mlir::acc::ReductionRecipeOp recipe) {
2968 op.setRecipeAttr(mlir::SymbolRefAttr::get(ctx: context, value: recipe.getSymName()));
2969 getReductionOperandsMutable().append(values: op.getResult());
2970}
2971
2972void acc::KernelsOp::addNumWorkersOperand(
2973 MLIRContext *context, mlir::Value newValue,
2974 llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
2975 setNumWorkersDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(
2976 context, existingDeviceTypes: getNumWorkersDeviceTypeAttr(), newDeviceTypes: effectiveDeviceTypes, arguments: newValue,
2977 argCollection: getNumWorkersMutable()));
2978}
2979
2980void acc::KernelsOp::addVectorLengthOperand(
2981 MLIRContext *context, mlir::Value newValue,
2982 llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
2983 setVectorLengthDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(
2984 context, existingDeviceTypes: getVectorLengthDeviceTypeAttr(), newDeviceTypes: effectiveDeviceTypes, arguments: newValue,
2985 argCollection: getVectorLengthMutable()));
2986}
2987void acc::KernelsOp::addAsyncOnly(
2988 MLIRContext *context, llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
2989 setAsyncOnlyAttr(addDeviceTypeAffectedOperandHelper(
2990 context, existingDeviceTypes: getAsyncOnlyAttr(), newDeviceTypes: effectiveDeviceTypes));
2991}
2992
2993void acc::KernelsOp::addAsyncOperand(
2994 MLIRContext *context, mlir::Value newValue,
2995 llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
2996 setAsyncOperandsDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(
2997 context, existingDeviceTypes: getAsyncOperandsDeviceTypeAttr(), newDeviceTypes: effectiveDeviceTypes, arguments: newValue,
2998 argCollection: getAsyncOperandsMutable()));
2999}
3000
3001void acc::KernelsOp::addNumGangsOperands(
3002 MLIRContext *context, mlir::ValueRange newValues,
3003 llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
3004 llvm::SmallVector<int32_t> segments;
3005 if (getNumGangsSegmentsAttr())
3006 llvm::copy(Range: *getNumGangsSegments(), Out: std::back_inserter(x&: segments));
3007
3008 setNumGangsDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(
3009 context, existingDeviceTypes: getNumGangsDeviceTypeAttr(), newDeviceTypes: effectiveDeviceTypes, arguments: newValues,
3010 argCollection: getNumGangsMutable(), segments));
3011
3012 setNumGangsSegments(segments);
3013}
3014
3015void acc::KernelsOp::addWaitOnly(
3016 MLIRContext *context, llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
3017 setWaitOnlyAttr(addDeviceTypeAffectedOperandHelper(context, existingDeviceTypes: getWaitOnlyAttr(),
3018 newDeviceTypes: effectiveDeviceTypes));
3019}
3020void acc::KernelsOp::addWaitOperands(
3021 MLIRContext *context, bool hasDevnum, mlir::ValueRange newValues,
3022 llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
3023
3024 llvm::SmallVector<int32_t> segments;
3025 if (getWaitOperandsSegments())
3026 llvm::copy(Range: *getWaitOperandsSegments(), Out: std::back_inserter(x&: segments));
3027
3028 setWaitOperandsDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(
3029 context, existingDeviceTypes: getWaitOperandsDeviceTypeAttr(), newDeviceTypes: effectiveDeviceTypes, arguments: newValues,
3030 argCollection: getWaitOperandsMutable(), segments));
3031 setWaitOperandsSegments(segments);
3032
3033 llvm::SmallVector<mlir::Attribute> hasDevnums;
3034 if (getHasWaitDevnumAttr())
3035 llvm::copy(Range: getHasWaitDevnumAttr(), Out: std::back_inserter(x&: hasDevnums));
3036 hasDevnums.insert(
3037 I: hasDevnums.end(),
3038 NumToInsert: std::max(a: effectiveDeviceTypes.size(), b: static_cast<size_t>(1)),
3039 Elt: mlir::BoolAttr::get(context, value: hasDevnum));
3040 setHasWaitDevnumAttr(mlir::ArrayAttr::get(context, value: hasDevnums));
3041}
3042
3043//===----------------------------------------------------------------------===//
3044// HostDataOp
3045//===----------------------------------------------------------------------===//
3046
3047LogicalResult acc::HostDataOp::verify() {
3048 if (getDataClauseOperands().empty())
3049 return emitError(message: "at least one operand must appear on the host_data "
3050 "operation");
3051
3052 llvm::SmallPtrSet<mlir::Value, 4> seenVars;
3053 for (mlir::Value operand : getDataClauseOperands()) {
3054 auto useDeviceOp =
3055 mlir::dyn_cast_if_present<acc::UseDeviceOp>(Val: operand.getDefiningOp());
3056 if (!useDeviceOp)
3057 return emitError(message: "expect data entry operation as defining op");
3058
3059 // Check for duplicate use_device clauses
3060 if (!seenVars.insert(Ptr: useDeviceOp.getVar()).second)
3061 return emitError(message: "duplicate use_device variable");
3062 }
3063 return success();
3064}
3065
3066void acc::HostDataOp::getCanonicalizationPatterns(RewritePatternSet &results,
3067 MLIRContext *context) {
3068 results.add<RemoveConstantIfConditionWithRegion<HostDataOp>>(arg&: context);
3069}
3070
3071//===----------------------------------------------------------------------===//
3072// LoopOp
3073//===----------------------------------------------------------------------===//
3074
3075static ParseResult parseGangValue(
3076 OpAsmParser &parser, llvm::StringRef keyword,
3077 llvm::SmallVectorImpl<mlir::OpAsmParser::UnresolvedOperand> &operands,
3078 llvm::SmallVectorImpl<Type> &types,
3079 llvm::SmallVector<GangArgTypeAttr> &attributes, GangArgTypeAttr gangArgType,
3080 bool &needCommaBetweenValues, bool &newValue) {
3081 if (succeeded(Result: parser.parseOptionalKeyword(keyword))) {
3082 if (parser.parseEqual())
3083 return failure();
3084 if (parser.parseOperand(result&: operands.emplace_back()) ||
3085 parser.parseColonType(result&: types.emplace_back()))
3086 return failure();
3087 attributes.push_back(Elt: gangArgType);
3088 needCommaBetweenValues = true;
3089 newValue = true;
3090 }
3091 return success();
3092}
3093
3094static ParseResult parseGangClause(
3095 OpAsmParser &parser,
3096 llvm::SmallVectorImpl<mlir::OpAsmParser::UnresolvedOperand> &gangOperands,
3097 llvm::SmallVectorImpl<Type> &gangOperandsType, mlir::ArrayAttr &gangArgType,
3098 mlir::ArrayAttr &deviceType, mlir::DenseI32ArrayAttr &segments,
3099 mlir::ArrayAttr &gangOnlyDeviceType) {
3100 llvm::SmallVector<GangArgTypeAttr> gangArgTypeAttributes;
3101 llvm::SmallVector<mlir::Attribute> deviceTypeAttributes;
3102 llvm::SmallVector<mlir::Attribute> gangOnlyDeviceTypeAttributes;
3103 llvm::SmallVector<int32_t> seg;
3104 bool needCommaBetweenValues = false;
3105 bool needCommaBeforeOperands = false;
3106
3107 if (failed(Result: parser.parseOptionalLParen())) {
3108 // Gang only keyword
3109 gangOnlyDeviceTypeAttributes.push_back(Elt: mlir::acc::DeviceTypeAttr::get(
3110 context: parser.getContext(), value: mlir::acc::DeviceType::None));
3111 gangOnlyDeviceType =
3112 ArrayAttr::get(context: parser.getContext(), value: gangOnlyDeviceTypeAttributes);
3113 return success();
3114 }
3115
3116 // Parse gang only attributes
3117 if (succeeded(Result: parser.parseOptionalLSquare())) {
3118 // Parse gang only attributes
3119 if (failed(Result: parser.parseCommaSeparatedList(parseElementFn: [&]() {
3120 if (parser.parseAttribute(
3121 result&: gangOnlyDeviceTypeAttributes.emplace_back()))
3122 return failure();
3123 return success();
3124 })))
3125 return failure();
3126 if (parser.parseRSquare())
3127 return failure();
3128 needCommaBeforeOperands = true;
3129 }
3130
3131 auto argNum = mlir::acc::GangArgTypeAttr::get(context: parser.getContext(),
3132 value: mlir::acc::GangArgType::Num);
3133 auto argDim = mlir::acc::GangArgTypeAttr::get(context: parser.getContext(),
3134 value: mlir::acc::GangArgType::Dim);
3135 auto argStatic = mlir::acc::GangArgTypeAttr::get(
3136 context: parser.getContext(), value: mlir::acc::GangArgType::Static);
3137
3138 do {
3139 if (needCommaBeforeOperands) {
3140 needCommaBeforeOperands = false;
3141 continue;
3142 }
3143
3144 if (failed(Result: parser.parseLBrace()))
3145 return failure();
3146
3147 int32_t crtOperandsSize = gangOperands.size();
3148 while (true) {
3149 bool newValue = false;
3150 bool needValue = false;
3151 if (needCommaBetweenValues) {
3152 if (succeeded(Result: parser.parseOptionalComma()))
3153 needValue = true; // expect a new value after comma.
3154 else
3155 break;
3156 }
3157
3158 if (failed(Result: parseGangValue(parser, keyword: LoopOp::getGangNumKeyword(),
3159 operands&: gangOperands, types&: gangOperandsType,
3160 attributes&: gangArgTypeAttributes, gangArgType: argNum,
3161 needCommaBetweenValues, newValue)))
3162 return failure();
3163 if (failed(Result: parseGangValue(parser, keyword: LoopOp::getGangDimKeyword(),
3164 operands&: gangOperands, types&: gangOperandsType,
3165 attributes&: gangArgTypeAttributes, gangArgType: argDim,
3166 needCommaBetweenValues, newValue)))
3167 return failure();
3168 if (failed(Result: parseGangValue(parser, keyword: LoopOp::getGangStaticKeyword(),
3169 operands&: gangOperands, types&: gangOperandsType,
3170 attributes&: gangArgTypeAttributes, gangArgType: argStatic,
3171 needCommaBetweenValues, newValue)))
3172 return failure();
3173
3174 if (!newValue && needValue) {
3175 parser.emitError(loc: parser.getCurrentLocation(),
3176 message: "new value expected after comma");
3177 return failure();
3178 }
3179
3180 if (!newValue)
3181 break;
3182 }
3183
3184 if (gangOperands.empty())
3185 return parser.emitError(
3186 loc: parser.getCurrentLocation(),
3187 message: "expect at least one of num, dim or static values");
3188
3189 if (failed(Result: parser.parseRBrace()))
3190 return failure();
3191
3192 if (succeeded(Result: parser.parseOptionalLSquare())) {
3193 if (parser.parseAttribute(result&: deviceTypeAttributes.emplace_back()) ||
3194 parser.parseRSquare())
3195 return failure();
3196 } else {
3197 deviceTypeAttributes.push_back(Elt: mlir::acc::DeviceTypeAttr::get(
3198 context: parser.getContext(), value: mlir::acc::DeviceType::None));
3199 }
3200
3201 seg.push_back(Elt: gangOperands.size() - crtOperandsSize);
3202
3203 } while (succeeded(Result: parser.parseOptionalComma()));
3204
3205 if (failed(Result: parser.parseRParen()))
3206 return failure();
3207
3208 llvm::SmallVector<mlir::Attribute> arrayAttr(gangArgTypeAttributes.begin(),
3209 gangArgTypeAttributes.end());
3210 gangArgType = ArrayAttr::get(context: parser.getContext(), value: arrayAttr);
3211 deviceType = ArrayAttr::get(context: parser.getContext(), value: deviceTypeAttributes);
3212
3213 llvm::SmallVector<mlir::Attribute> gangOnlyAttr(
3214 gangOnlyDeviceTypeAttributes.begin(), gangOnlyDeviceTypeAttributes.end());
3215 gangOnlyDeviceType = ArrayAttr::get(context: parser.getContext(), value: gangOnlyAttr);
3216
3217 segments = DenseI32ArrayAttr::get(context: parser.getContext(), content: seg);
3218 return success();
3219}
3220
3221void printGangClause(OpAsmPrinter &p, Operation *op,
3222 mlir::OperandRange operands, mlir::TypeRange types,
3223 std::optional<mlir::ArrayAttr> gangArgTypes,
3224 std::optional<mlir::ArrayAttr> deviceTypes,
3225 std::optional<mlir::DenseI32ArrayAttr> segments,
3226 std::optional<mlir::ArrayAttr> gangOnlyDeviceTypes) {
3227
3228 if (operands.begin() == operands.end() &&
3229 hasOnlyDeviceTypeNone(attrs: gangOnlyDeviceTypes)) {
3230 return;
3231 }
3232
3233 p << "(";
3234
3235 printDeviceTypes(p, deviceTypes: gangOnlyDeviceTypes);
3236
3237 if (hasDeviceTypeValues(arrayAttr: gangOnlyDeviceTypes) &&
3238 hasDeviceTypeValues(arrayAttr: deviceTypes))
3239 p << ", ";
3240
3241 if (hasDeviceTypeValues(arrayAttr: deviceTypes)) {
3242 unsigned opIdx = 0;
3243 llvm::interleaveComma(c: llvm::enumerate(First&: *deviceTypes), os&: p, each_fn: [&](auto it) {
3244 p << "{";
3245 llvm::interleaveComma(
3246 llvm::seq<int32_t>(0, (*segments)[it.index()]), p, [&](auto it) {
3247 auto gangArgTypeAttr = mlir::dyn_cast<mlir::acc::GangArgTypeAttr>(
3248 Val: (*gangArgTypes)[opIdx]);
3249 if (gangArgTypeAttr.getValue() == mlir::acc::GangArgType::Num)
3250 p << LoopOp::getGangNumKeyword();
3251 else if (gangArgTypeAttr.getValue() == mlir::acc::GangArgType::Dim)
3252 p << LoopOp::getGangDimKeyword();
3253 else if (gangArgTypeAttr.getValue() ==
3254 mlir::acc::GangArgType::Static)
3255 p << LoopOp::getGangStaticKeyword();
3256 p << "=" << operands[opIdx] << " : " << operands[opIdx].getType();
3257 ++opIdx;
3258 });
3259 p << "}";
3260 printSingleDeviceType(p, it.value());
3261 });
3262 }
3263 p << ")";
3264}
3265
3266bool hasDuplicateDeviceTypes(
3267 std::optional<mlir::ArrayAttr> segments,
3268 llvm::SmallSet<mlir::acc::DeviceType, 3> &deviceTypes) {
3269 if (!segments)
3270 return false;
3271 for (auto attr : *segments) {
3272 auto deviceTypeAttr = mlir::dyn_cast<mlir::acc::DeviceTypeAttr>(Val&: attr);
3273 if (!deviceTypes.insert(V: deviceTypeAttr.getValue()).second)
3274 return true;
3275 }
3276 return false;
3277}
3278
3279/// Check for duplicates in the DeviceType array attribute.
3280/// Returns std::nullopt if no duplicates, or the duplicate DeviceType if found.
3281static std::optional<mlir::acc::DeviceType>
3282checkDeviceTypes(mlir::ArrayAttr deviceTypes) {
3283 llvm::SmallSet<mlir::acc::DeviceType, 3> crtDeviceTypes;
3284 if (!deviceTypes)
3285 return std::nullopt;
3286 for (auto attr : deviceTypes) {
3287 auto deviceTypeAttr =
3288 mlir::dyn_cast_or_null<mlir::acc::DeviceTypeAttr>(Val&: attr);
3289 if (!deviceTypeAttr)
3290 return mlir::acc::DeviceType::None;
3291 if (!crtDeviceTypes.insert(V: deviceTypeAttr.getValue()).second)
3292 return deviceTypeAttr.getValue();
3293 }
3294 return std::nullopt;
3295}
3296
3297LogicalResult acc::LoopOp::verify() {
3298 if (getUpperbound().size() != getStep().size())
3299 return emitError() << "number of upperbounds expected to be the same as "
3300 "number of steps";
3301
3302 if (getUpperbound().size() != getLowerbound().size())
3303 return emitError() << "number of upperbounds expected to be the same as "
3304 "number of lowerbounds";
3305
3306 if (!getUpperbound().empty() && getInclusiveUpperbound() &&
3307 (getUpperbound().size() != getInclusiveUpperbound()->size()))
3308 return emitError() << "inclusiveUpperbound size is expected to be the same"
3309 << " as upperbound size";
3310
3311 // Check collapse
3312 if (getCollapseAttr() && !getCollapseDeviceTypeAttr())
3313 return emitOpError() << "collapse device_type attr must be define when"
3314 << " collapse attr is present";
3315
3316 if (getCollapseAttr() && getCollapseDeviceTypeAttr() &&
3317 getCollapseAttr().getValue().size() !=
3318 getCollapseDeviceTypeAttr().getValue().size())
3319 return emitOpError() << "collapse attribute count must match collapse"
3320 << " device_type count";
3321 if (auto duplicateDeviceType = checkDeviceTypes(deviceTypes: getCollapseDeviceTypeAttr()))
3322 return emitOpError() << "duplicate device_type `"
3323 << acc::stringifyDeviceType(val: *duplicateDeviceType)
3324 << "` found in collapseDeviceType attribute";
3325
3326 // Check gang
3327 if (!getGangOperands().empty()) {
3328 if (!getGangOperandsArgType())
3329 return emitOpError() << "gangOperandsArgType attribute must be defined"
3330 << " when gang operands are present";
3331
3332 if (getGangOperands().size() !=
3333 getGangOperandsArgTypeAttr().getValue().size())
3334 return emitOpError() << "gangOperandsArgType attribute count must match"
3335 << " gangOperands count";
3336 }
3337 if (getGangAttr()) {
3338 if (auto duplicateDeviceType = checkDeviceTypes(deviceTypes: getGangAttr()))
3339 return emitOpError() << "duplicate device_type `"
3340 << acc::stringifyDeviceType(val: *duplicateDeviceType)
3341 << "` found in gang attribute";
3342 }
3343
3344 if (failed(Result: verifyDeviceTypeAndSegmentCountMatch(
3345 op: *this, operands: getGangOperands(), segments: getGangOperandsSegmentsAttr(),
3346 deviceTypes: getGangOperandsDeviceTypeAttr(), keyword: "gang")))
3347 return failure();
3348
3349 // Check worker
3350 if (auto duplicateDeviceType = checkDeviceTypes(deviceTypes: getWorkerAttr()))
3351 return emitOpError() << "duplicate device_type `"
3352 << acc::stringifyDeviceType(val: *duplicateDeviceType)
3353 << "` found in worker attribute";
3354 if (auto duplicateDeviceType =
3355 checkDeviceTypes(deviceTypes: getWorkerNumOperandsDeviceTypeAttr()))
3356 return emitOpError() << "duplicate device_type `"
3357 << acc::stringifyDeviceType(val: *duplicateDeviceType)
3358 << "` found in workerNumOperandsDeviceType attribute";
3359 if (failed(Result: verifyDeviceTypeCountMatch(op: *this, operands: getWorkerNumOperands(),
3360 deviceTypes: getWorkerNumOperandsDeviceTypeAttr(),
3361 keyword: "worker")))
3362 return failure();
3363
3364 // Check vector
3365 if (auto duplicateDeviceType = checkDeviceTypes(deviceTypes: getVectorAttr()))
3366 return emitOpError() << "duplicate device_type `"
3367 << acc::stringifyDeviceType(val: *duplicateDeviceType)
3368 << "` found in vector attribute";
3369 if (auto duplicateDeviceType =
3370 checkDeviceTypes(deviceTypes: getVectorOperandsDeviceTypeAttr()))
3371 return emitOpError() << "duplicate device_type `"
3372 << acc::stringifyDeviceType(val: *duplicateDeviceType)
3373 << "` found in vectorOperandsDeviceType attribute";
3374 if (failed(Result: verifyDeviceTypeCountMatch(op: *this, operands: getVectorOperands(),
3375 deviceTypes: getVectorOperandsDeviceTypeAttr(),
3376 keyword: "vector")))
3377 return failure();
3378
3379 if (failed(Result: verifyDeviceTypeAndSegmentCountMatch(
3380 op: *this, operands: getTileOperands(), segments: getTileOperandsSegmentsAttr(),
3381 deviceTypes: getTileOperandsDeviceTypeAttr(), keyword: "tile")))
3382 return failure();
3383
3384 // auto, independent and seq attribute are mutually exclusive.
3385 llvm::SmallSet<mlir::acc::DeviceType, 3> deviceTypes;
3386 if (hasDuplicateDeviceTypes(segments: getAuto_(), deviceTypes) ||
3387 hasDuplicateDeviceTypes(segments: getIndependent(), deviceTypes) ||
3388 hasDuplicateDeviceTypes(segments: getSeq(), deviceTypes)) {
3389 return emitError() << "only one of auto, independent, seq can be present "
3390 "at the same time";
3391 }
3392
3393 // Check that at least one of auto, independent, or seq is present
3394 // for the device-independent default clauses.
3395 auto hasDeviceNone = [](mlir::acc::DeviceTypeAttr attr) -> bool {
3396 return attr.getValue() == mlir::acc::DeviceType::None;
3397 };
3398 bool hasDefaultSeq =
3399 getSeqAttr()
3400 ? llvm::any_of(Range: getSeqAttr().getAsRange<mlir::acc::DeviceTypeAttr>(),
3401 P: hasDeviceNone)
3402 : false;
3403 bool hasDefaultIndependent =
3404 getIndependentAttr()
3405 ? llvm::any_of(
3406 Range: getIndependentAttr().getAsRange<mlir::acc::DeviceTypeAttr>(),
3407 P: hasDeviceNone)
3408 : false;
3409 bool hasDefaultAuto =
3410 getAuto_Attr()
3411 ? llvm::any_of(Range: getAuto_Attr().getAsRange<mlir::acc::DeviceTypeAttr>(),
3412 P: hasDeviceNone)
3413 : false;
3414 if (!hasDefaultSeq && !hasDefaultIndependent && !hasDefaultAuto) {
3415 return emitError()
3416 << "at least one of auto, independent, seq must be present";
3417 }
3418
3419 // Gang, worker and vector are incompatible with seq.
3420 if (getSeqAttr()) {
3421 for (auto attr : getSeqAttr()) {
3422 auto deviceTypeAttr = mlir::dyn_cast<mlir::acc::DeviceTypeAttr>(Val&: attr);
3423 if (hasVector(deviceType: deviceTypeAttr.getValue()) ||
3424 getVectorValue(deviceType: deviceTypeAttr.getValue()) ||
3425 hasWorker(deviceType: deviceTypeAttr.getValue()) ||
3426 getWorkerValue(deviceType: deviceTypeAttr.getValue()) ||
3427 hasGang(deviceType: deviceTypeAttr.getValue()) ||
3428 getGangValue(gangArgType: mlir::acc::GangArgType::Num,
3429 deviceType: deviceTypeAttr.getValue()) ||
3430 getGangValue(gangArgType: mlir::acc::GangArgType::Dim,
3431 deviceType: deviceTypeAttr.getValue()) ||
3432 getGangValue(gangArgType: mlir::acc::GangArgType::Static,
3433 deviceType: deviceTypeAttr.getValue()))
3434 return emitError() << "gang, worker or vector cannot appear with seq";
3435 }
3436 }
3437
3438 if (failed(Result: checkPrivateOperands<mlir::acc::PrivateOp,
3439 mlir::acc::PrivateRecipeOp>(
3440 accConstructOp: *this, operands: getPrivateOperands(), operandName: "private")))
3441 return failure();
3442
3443 if (failed(Result: checkPrivateOperands<mlir::acc::FirstprivateOp,
3444 mlir::acc::FirstprivateRecipeOp>(
3445 accConstructOp: *this, operands: getFirstprivateOperands(), operandName: "firstprivate")))
3446 return failure();
3447
3448 if (failed(Result: checkPrivateOperands<mlir::acc::ReductionOp,
3449 mlir::acc::ReductionRecipeOp>(
3450 accConstructOp: *this, operands: getReductionOperands(), operandName: "reduction")))
3451 return failure();
3452
3453 if (getCombined().has_value() &&
3454 (getCombined().value() != acc::CombinedConstructsType::ParallelLoop &&
3455 getCombined().value() != acc::CombinedConstructsType::KernelsLoop &&
3456 getCombined().value() != acc::CombinedConstructsType::SerialLoop)) {
3457 return emitError(message: "unexpected combined constructs attribute");
3458 }
3459
3460 // Check non-empty body().
3461 if (getRegion().empty())
3462 return emitError(message: "expected non-empty body.");
3463
3464 if (getUnstructured()) {
3465 if (!isContainerLike())
3466 return emitError(
3467 message: "unstructured acc.loop must not have induction variables");
3468 } else if (isContainerLike()) {
3469 // When it is container-like - it is expected to hold a loop-like operation.
3470 // Obtain the maximum collapse count - we use this to check that there
3471 // are enough loops contained.
3472 uint64_t collapseCount = getCollapseValue().value_or(u: 1);
3473 if (getCollapseAttr()) {
3474 for (auto collapseEntry : getCollapseAttr()) {
3475 auto intAttr = mlir::dyn_cast<IntegerAttr>(Val&: collapseEntry);
3476 if (intAttr.getValue().getZExtValue() > collapseCount)
3477 collapseCount = intAttr.getValue().getZExtValue();
3478 }
3479 }
3480
3481 // We want to check that we find enough loop-like operations inside.
3482 // PreOrder walk allows us to walk in a breadth-first manner at each nesting
3483 // level.
3484 mlir::Operation *expectedParent = this->getOperation();
3485 bool foundSibling = false;
3486 getRegion().walk<WalkOrder::PreOrder>(callback: [&](mlir::Operation *op) {
3487 if (mlir::isa<mlir::LoopLikeOpInterface>(Val: op)) {
3488 // This effectively checks that we are not looking at a sibling loop.
3489 if (op->getParentOfType<mlir::LoopLikeOpInterface>() !=
3490 expectedParent) {
3491 foundSibling = true;
3492 return mlir::WalkResult::interrupt();
3493 }
3494
3495 collapseCount--;
3496 expectedParent = op;
3497 }
3498 // We found enough contained loops.
3499 if (collapseCount == 0)
3500 return mlir::WalkResult::interrupt();
3501 return mlir::WalkResult::advance();
3502 });
3503
3504 if (foundSibling)
3505 return emitError(message: "found sibling loops inside container-like acc.loop");
3506 if (collapseCount != 0)
3507 return emitError(message: "failed to find enough loop-like operations inside "
3508 "container-like acc.loop");
3509 }
3510
3511 return success();
3512}
3513
3514unsigned LoopOp::getNumDataOperands() {
3515 return getReductionOperands().size() + getPrivateOperands().size() +
3516 getFirstprivateOperands().size();
3517}
3518
3519Value LoopOp::getDataOperand(unsigned i) {
3520 unsigned numOptional =
3521 getLowerbound().size() + getUpperbound().size() + getStep().size();
3522 numOptional += getGangOperands().size();
3523 numOptional += getVectorOperands().size();
3524 numOptional += getWorkerNumOperands().size();
3525 numOptional += getTileOperands().size();
3526 numOptional += getCacheOperands().size();
3527 return getOperand(i: numOptional + i);
3528}
3529
3530bool LoopOp::hasAuto() { return hasAuto(deviceType: mlir::acc::DeviceType::None); }
3531
3532bool LoopOp::hasAuto(mlir::acc::DeviceType deviceType) {
3533 return hasDeviceType(arrayAttr: getAuto_(), deviceType);
3534}
3535
3536bool LoopOp::hasIndependent() {
3537 return hasIndependent(deviceType: mlir::acc::DeviceType::None);
3538}
3539
3540bool LoopOp::hasIndependent(mlir::acc::DeviceType deviceType) {
3541 return hasDeviceType(arrayAttr: getIndependent(), deviceType);
3542}
3543
3544bool LoopOp::hasSeq() { return hasSeq(deviceType: mlir::acc::DeviceType::None); }
3545
3546bool LoopOp::hasSeq(mlir::acc::DeviceType deviceType) {
3547 return hasDeviceType(arrayAttr: getSeq(), deviceType);
3548}
3549
3550mlir::Value LoopOp::getVectorValue() {
3551 return getVectorValue(deviceType: mlir::acc::DeviceType::None);
3552}
3553
3554mlir::Value LoopOp::getVectorValue(mlir::acc::DeviceType deviceType) {
3555 return getValueInDeviceTypeSegment(arrayAttr: getVectorOperandsDeviceType(),
3556 range: getVectorOperands(), deviceType);
3557}
3558
3559bool LoopOp::hasVector() { return hasVector(deviceType: mlir::acc::DeviceType::None); }
3560
3561bool LoopOp::hasVector(mlir::acc::DeviceType deviceType) {
3562 return hasDeviceType(arrayAttr: getVector(), deviceType);
3563}
3564
3565mlir::Value LoopOp::getWorkerValue() {
3566 return getWorkerValue(deviceType: mlir::acc::DeviceType::None);
3567}
3568
3569mlir::Value LoopOp::getWorkerValue(mlir::acc::DeviceType deviceType) {
3570 return getValueInDeviceTypeSegment(arrayAttr: getWorkerNumOperandsDeviceType(),
3571 range: getWorkerNumOperands(), deviceType);
3572}
3573
3574bool LoopOp::hasWorker() { return hasWorker(deviceType: mlir::acc::DeviceType::None); }
3575
3576bool LoopOp::hasWorker(mlir::acc::DeviceType deviceType) {
3577 return hasDeviceType(arrayAttr: getWorker(), deviceType);
3578}
3579
3580mlir::Operation::operand_range LoopOp::getTileValues() {
3581 return getTileValues(deviceType: mlir::acc::DeviceType::None);
3582}
3583
3584mlir::Operation::operand_range
3585LoopOp::getTileValues(mlir::acc::DeviceType deviceType) {
3586 return getValuesFromSegments(arrayAttr: getTileOperandsDeviceType(), range: getTileOperands(),
3587 segments: getTileOperandsSegments(), deviceType);
3588}
3589
3590std::optional<int64_t> LoopOp::getCollapseValue() {
3591 return getCollapseValue(deviceType: mlir::acc::DeviceType::None);
3592}
3593
3594std::optional<int64_t>
3595LoopOp::getCollapseValue(mlir::acc::DeviceType deviceType) {
3596 if (!getCollapseAttr())
3597 return std::nullopt;
3598 if (auto pos = findSegment(segments: getCollapseDeviceTypeAttr(), deviceType)) {
3599 auto intAttr =
3600 mlir::dyn_cast<IntegerAttr>(Val: getCollapseAttr().getValue()[*pos]);
3601 return intAttr.getValue().getZExtValue();
3602 }
3603 return std::nullopt;
3604}
3605
3606mlir::Value LoopOp::getGangValue(mlir::acc::GangArgType gangArgType) {
3607 return getGangValue(gangArgType, deviceType: mlir::acc::DeviceType::None);
3608}
3609
3610mlir::Value LoopOp::getGangValue(mlir::acc::GangArgType gangArgType,
3611 mlir::acc::DeviceType deviceType) {
3612 if (getGangOperands().empty())
3613 return {};
3614 if (auto pos = findSegment(segments: *getGangOperandsDeviceType(), deviceType)) {
3615 int32_t nbOperandsBefore = 0;
3616 for (unsigned i = 0; i < *pos; ++i)
3617 nbOperandsBefore += (*getGangOperandsSegments())[i];
3618 mlir::Operation::operand_range values =
3619 getGangOperands()
3620 .drop_front(n: nbOperandsBefore)
3621 .take_front(n: (*getGangOperandsSegments())[*pos]);
3622
3623 int32_t argTypeIdx = nbOperandsBefore;
3624 for (auto value : values) {
3625 auto gangArgTypeAttr = mlir::dyn_cast<mlir::acc::GangArgTypeAttr>(
3626 Val: (*getGangOperandsArgType())[argTypeIdx]);
3627 if (gangArgTypeAttr.getValue() == gangArgType)
3628 return value;
3629 ++argTypeIdx;
3630 }
3631 }
3632 return {};
3633}
3634
3635bool LoopOp::hasGang() { return hasGang(deviceType: mlir::acc::DeviceType::None); }
3636
3637bool LoopOp::hasGang(mlir::acc::DeviceType deviceType) {
3638 return hasDeviceType(arrayAttr: getGang(), deviceType);
3639}
3640
3641llvm::SmallVector<mlir::Region *> acc::LoopOp::getLoopRegions() {
3642 return {&getRegion()};
3643}
3644
3645/// loop-control ::= `control` `(` ssa-id-and-type-list `)` `=`
3646/// `(` ssa-id-and-type-list `)` `to` `(` ssa-id-and-type-list `)` `step`
3647/// `(` ssa-id-and-type-list `)`
3648/// region
3649ParseResult
3650parseLoopControl(OpAsmParser &parser, Region &region,
3651 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &lowerbound,
3652 SmallVectorImpl<Type> &lowerboundType,
3653 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &upperbound,
3654 SmallVectorImpl<Type> &upperboundType,
3655 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &step,
3656 SmallVectorImpl<Type> &stepType) {
3657
3658 SmallVector<OpAsmParser::Argument> inductionVars;
3659 if (succeeded(
3660 Result: parser.parseOptionalKeyword(keyword: acc::LoopOp::getControlKeyword()))) {
3661 if (parser.parseLParen() ||
3662 parser.parseArgumentList(result&: inductionVars, delimiter: OpAsmParser::Delimiter::None,
3663 /*allowType=*/true) ||
3664 parser.parseRParen() || parser.parseEqual() || parser.parseLParen() ||
3665 parser.parseOperandList(result&: lowerbound, requiredOperandCount: inductionVars.size(),
3666 delimiter: OpAsmParser::Delimiter::None) ||
3667 parser.parseColonTypeList(result&: lowerboundType) || parser.parseRParen() ||
3668 parser.parseKeyword(keyword: "to") || parser.parseLParen() ||
3669 parser.parseOperandList(result&: upperbound, requiredOperandCount: inductionVars.size(),
3670 delimiter: OpAsmParser::Delimiter::None) ||
3671 parser.parseColonTypeList(result&: upperboundType) || parser.parseRParen() ||
3672 parser.parseKeyword(keyword: "step") || parser.parseLParen() ||
3673 parser.parseOperandList(result&: step, requiredOperandCount: inductionVars.size(),
3674 delimiter: OpAsmParser::Delimiter::None) ||
3675 parser.parseColonTypeList(result&: stepType) || parser.parseRParen())
3676 return failure();
3677 }
3678 return parser.parseRegion(region, arguments: inductionVars);
3679}
3680
3681void printLoopControl(OpAsmPrinter &p, Operation *op, Region &region,
3682 ValueRange lowerbound, TypeRange lowerboundType,
3683 ValueRange upperbound, TypeRange upperboundType,
3684 ValueRange steps, TypeRange stepType) {
3685 ValueRange regionArgs = region.front().getArguments();
3686 if (!regionArgs.empty()) {
3687 p << acc::LoopOp::getControlKeyword() << "(";
3688 llvm::interleaveComma(c: regionArgs, os&: p,
3689 each_fn: [&p](Value v) { p << v << " : " << v.getType(); });
3690 p << ") = (" << lowerbound << " : " << lowerboundType << ") to ("
3691 << upperbound << " : " << upperboundType << ") " << " step (" << steps
3692 << " : " << stepType << ") ";
3693 }
3694 p.printRegion(blocks&: region, /*printEntryBlockArgs=*/false);
3695}
3696
3697void acc::LoopOp::addSeq(MLIRContext *context,
3698 llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
3699 setSeqAttr(addDeviceTypeAffectedOperandHelper(context, existingDeviceTypes: getSeqAttr(),
3700 newDeviceTypes: effectiveDeviceTypes));
3701}
3702
3703void acc::LoopOp::addIndependent(
3704 MLIRContext *context, llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
3705 setIndependentAttr(addDeviceTypeAffectedOperandHelper(
3706 context, existingDeviceTypes: getIndependentAttr(), newDeviceTypes: effectiveDeviceTypes));
3707}
3708
3709void acc::LoopOp::addAuto(MLIRContext *context,
3710 llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
3711 setAuto_Attr(addDeviceTypeAffectedOperandHelper(context, existingDeviceTypes: getAuto_Attr(),
3712 newDeviceTypes: effectiveDeviceTypes));
3713}
3714
3715void acc::LoopOp::setCollapseForDeviceTypes(
3716 MLIRContext *context, llvm::ArrayRef<DeviceType> effectiveDeviceTypes,
3717 llvm::APInt value) {
3718 llvm::SmallVector<mlir::Attribute> newValues;
3719 llvm::SmallVector<mlir::Attribute> newDeviceTypes;
3720
3721 assert((getCollapseAttr() == nullptr) ==
3722 (getCollapseDeviceTypeAttr() == nullptr));
3723 assert(value.getBitWidth() == 64);
3724
3725 if (getCollapseAttr()) {
3726 for (const auto &existing :
3727 llvm::zip_equal(t: getCollapseAttr(), u: getCollapseDeviceTypeAttr())) {
3728 newValues.push_back(Elt: std::get<0>(t: existing));
3729 newDeviceTypes.push_back(Elt: std::get<1>(t: existing));
3730 }
3731 }
3732
3733 if (effectiveDeviceTypes.empty()) {
3734 // If the effective device-types list is empty, this is before there are any
3735 // being applied by device_type, so this should be added as a 'none'.
3736 newValues.push_back(
3737 Elt: mlir::IntegerAttr::get(type: mlir::IntegerType::get(context, width: 64), value));
3738 newDeviceTypes.push_back(
3739 Elt: acc::DeviceTypeAttr::get(context, value: DeviceType::None));
3740 } else {
3741 for (DeviceType dt : effectiveDeviceTypes) {
3742 newValues.push_back(
3743 Elt: mlir::IntegerAttr::get(type: mlir::IntegerType::get(context, width: 64), value));
3744 newDeviceTypes.push_back(Elt: acc::DeviceTypeAttr::get(context, value: dt));
3745 }
3746 }
3747
3748 setCollapseAttr(ArrayAttr::get(context, value: newValues));
3749 setCollapseDeviceTypeAttr(ArrayAttr::get(context, value: newDeviceTypes));
3750}
3751
3752void acc::LoopOp::setTileForDeviceTypes(
3753 MLIRContext *context, llvm::ArrayRef<DeviceType> effectiveDeviceTypes,
3754 ValueRange values) {
3755 llvm::SmallVector<int32_t> segments;
3756 if (getTileOperandsSegments())
3757 llvm::copy(Range: *getTileOperandsSegments(), Out: std::back_inserter(x&: segments));
3758
3759 setTileOperandsDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(
3760 context, existingDeviceTypes: getTileOperandsDeviceTypeAttr(), newDeviceTypes: effectiveDeviceTypes, arguments: values,
3761 argCollection: getTileOperandsMutable(), segments));
3762
3763 setTileOperandsSegments(segments);
3764}
3765
3766void acc::LoopOp::addVectorOperand(
3767 MLIRContext *context, mlir::Value newValue,
3768 llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
3769 setVectorOperandsDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(
3770 context, existingDeviceTypes: getVectorOperandsDeviceTypeAttr(), newDeviceTypes: effectiveDeviceTypes,
3771 arguments: newValue, argCollection: getVectorOperandsMutable()));
3772}
3773
3774void acc::LoopOp::addEmptyVector(
3775 MLIRContext *context, llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
3776 setVectorAttr(addDeviceTypeAffectedOperandHelper(context, existingDeviceTypes: getVectorAttr(),
3777 newDeviceTypes: effectiveDeviceTypes));
3778}
3779
3780void acc::LoopOp::addWorkerNumOperand(
3781 MLIRContext *context, mlir::Value newValue,
3782 llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
3783 setWorkerNumOperandsDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(
3784 context, existingDeviceTypes: getWorkerNumOperandsDeviceTypeAttr(), newDeviceTypes: effectiveDeviceTypes,
3785 arguments: newValue, argCollection: getWorkerNumOperandsMutable()));
3786}
3787
3788void acc::LoopOp::addEmptyWorker(
3789 MLIRContext *context, llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
3790 setWorkerAttr(addDeviceTypeAffectedOperandHelper(context, existingDeviceTypes: getWorkerAttr(),
3791 newDeviceTypes: effectiveDeviceTypes));
3792}
3793
3794void acc::LoopOp::addEmptyGang(
3795 MLIRContext *context, llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
3796 setGangAttr(addDeviceTypeAffectedOperandHelper(context, existingDeviceTypes: getGangAttr(),
3797 newDeviceTypes: effectiveDeviceTypes));
3798}
3799
3800bool acc::LoopOp::hasParallelismFlag(DeviceType dt) {
3801 auto hasDevice = [=](DeviceTypeAttr attr) -> bool {
3802 return attr.getValue() == dt;
3803 };
3804 auto testFromArr = [=](ArrayAttr arr) -> bool {
3805 return llvm::any_of(Range: arr.getAsRange<DeviceTypeAttr>(), P: hasDevice);
3806 };
3807
3808 if (ArrayAttr arr = getSeqAttr(); arr && testFromArr(arr))
3809 return true;
3810 if (ArrayAttr arr = getIndependentAttr(); arr && testFromArr(arr))
3811 return true;
3812 if (ArrayAttr arr = getAuto_Attr(); arr && testFromArr(arr))
3813 return true;
3814
3815 return false;
3816}
3817
3818bool acc::LoopOp::hasDefaultGangWorkerVector() {
3819 return hasAnyGangWorkerVector(deviceType: DeviceType::None);
3820}
3821
3822bool acc::LoopOp::hasAnyGangWorkerVector(DeviceType deviceType) {
3823 return hasVector(deviceType) || getVectorValue(deviceType) ||
3824 hasWorker(deviceType) || getWorkerValue(deviceType) ||
3825 hasGang(deviceType) || getGangValue(gangArgType: GangArgType::Num, deviceType) ||
3826 getGangValue(gangArgType: GangArgType::Dim, deviceType) ||
3827 getGangValue(gangArgType: GangArgType::Static, deviceType);
3828}
3829
3830acc::LoopParMode
3831acc::LoopOp::getDefaultOrDeviceTypeParallelism(DeviceType deviceType) {
3832 if (hasSeq(deviceType))
3833 return LoopParMode::loop_seq;
3834 if (hasAuto(deviceType))
3835 return LoopParMode::loop_auto;
3836 if (hasIndependent(deviceType))
3837 return LoopParMode::loop_independent;
3838 if (hasSeq())
3839 return LoopParMode::loop_seq;
3840 if (hasAuto())
3841 return LoopParMode::loop_auto;
3842 assert(hasIndependent() &&
3843 "loop must have default auto, seq, or independent");
3844 return LoopParMode::loop_independent;
3845}
3846
3847void acc::LoopOp::addGangOperands(
3848 MLIRContext *context, llvm::ArrayRef<DeviceType> effectiveDeviceTypes,
3849 llvm::ArrayRef<GangArgType> argTypes, mlir::ValueRange values) {
3850 llvm::SmallVector<int32_t> segments;
3851 if (std::optional<ArrayRef<int32_t>> existingSegments =
3852 getGangOperandsSegments())
3853 llvm::copy(Range&: *existingSegments, Out: std::back_inserter(x&: segments));
3854
3855 unsigned beforeCount = segments.size();
3856
3857 setGangOperandsDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(
3858 context, existingDeviceTypes: getGangOperandsDeviceTypeAttr(), newDeviceTypes: effectiveDeviceTypes, arguments: values,
3859 argCollection: getGangOperandsMutable(), segments));
3860
3861 setGangOperandsSegments(segments);
3862
3863 // This is a bit of extra work to make sure we update the 'types' correctly by
3864 // adding to the types collection the correct number of times. We could
3865 // potentially add something similar to the
3866 // addDeviceTypeAffectedOperandHelper, but it seems that would be pretty
3867 // excessive for a one-off case.
3868 unsigned numAdded = segments.size() - beforeCount;
3869
3870 if (numAdded > 0) {
3871 llvm::SmallVector<mlir::Attribute> gangTypes;
3872 if (getGangOperandsArgTypeAttr())
3873 llvm::copy(Range: getGangOperandsArgTypeAttr(), Out: std::back_inserter(x&: gangTypes));
3874
3875 for (auto i : llvm::index_range(0u, numAdded)) {
3876 llvm::transform(Range&: argTypes, d_first: std::back_inserter(x&: gangTypes),
3877 F: [=](mlir::acc::GangArgType gangTy) {
3878 return mlir::acc::GangArgTypeAttr::get(context, value: gangTy);
3879 });
3880 (void)i;
3881 }
3882
3883 setGangOperandsArgTypeAttr(mlir::ArrayAttr::get(context, value: gangTypes));
3884 }
3885}
3886
3887void acc::LoopOp::addPrivatization(MLIRContext *context,
3888 mlir::acc::PrivateOp op,
3889 mlir::acc::PrivateRecipeOp recipe) {
3890 op.setRecipeAttr(mlir::SymbolRefAttr::get(ctx: context, value: recipe.getSymName()));
3891 getPrivateOperandsMutable().append(values: op.getResult());
3892}
3893
3894void acc::LoopOp::addFirstPrivatization(
3895 MLIRContext *context, mlir::acc::FirstprivateOp op,
3896 mlir::acc::FirstprivateRecipeOp recipe) {
3897 op.setRecipeAttr(mlir::SymbolRefAttr::get(ctx: context, value: recipe.getSymName()));
3898 getFirstprivateOperandsMutable().append(values: op.getResult());
3899}
3900
3901void acc::LoopOp::addReduction(MLIRContext *context, mlir::acc::ReductionOp op,
3902 mlir::acc::ReductionRecipeOp recipe) {
3903 op.setRecipeAttr(mlir::SymbolRefAttr::get(ctx: context, value: recipe.getSymName()));
3904 getReductionOperandsMutable().append(values: op.getResult());
3905}
3906
3907//===----------------------------------------------------------------------===//
3908// DataOp
3909//===----------------------------------------------------------------------===//
3910
3911LogicalResult acc::DataOp::verify() {
3912 // 2.6.5. Data Construct restriction
3913 // At least one copy, copyin, copyout, create, no_create, present, deviceptr,
3914 // attach, or default clause must appear on a data construct.
3915 if (getOperands().empty() && !getDefaultAttr())
3916 return emitError(message: "at least one operand or the default attribute "
3917 "must appear on the data operation");
3918
3919 for (mlir::Value operand : getDataClauseOperands())
3920 if (isa<BlockArgument>(Val: operand) ||
3921 !mlir::isa<acc::AttachOp, acc::CopyinOp, acc::CopyoutOp, acc::CreateOp,
3922 acc::DeleteOp, acc::DetachOp, acc::DevicePtrOp,
3923 acc::GetDevicePtrOp, acc::NoCreateOp, acc::PresentOp,
3924 acc::MapInfoOp>(Val: operand.getDefiningOp()))
3925 return emitError(message: "expect data entry/exit operation or acc.getdeviceptr "
3926 "as defining op");
3927
3928 if (failed(Result: checkWaitAndAsyncConflict<acc::DataOp>(op: *this)))
3929 return failure();
3930
3931 return success();
3932}
3933
3934unsigned DataOp::getNumDataOperands() { return getDataClauseOperands().size(); }
3935
3936Value DataOp::getDataOperand(unsigned i) {
3937 unsigned numOptional = getIfCond() ? 1 : 0;
3938 numOptional += getAsyncOperands().size() ? 1 : 0;
3939 numOptional += getWaitOperands().size();
3940 return getOperand(i: numOptional + i);
3941}
3942
3943bool acc::DataOp::hasAsyncOnly() {
3944 return hasAsyncOnly(deviceType: mlir::acc::DeviceType::None);
3945}
3946
3947bool acc::DataOp::hasAsyncOnly(mlir::acc::DeviceType deviceType) {
3948 return hasDeviceType(arrayAttr: getAsyncOnly(), deviceType);
3949}
3950
3951mlir::Value DataOp::getAsyncValue() {
3952 return getAsyncValue(deviceType: mlir::acc::DeviceType::None);
3953}
3954
3955mlir::Value DataOp::getAsyncValue(mlir::acc::DeviceType deviceType) {
3956 return getValueInDeviceTypeSegment(arrayAttr: getAsyncOperandsDeviceType(),
3957 range: getAsyncOperands(), deviceType);
3958}
3959
3960bool DataOp::hasWaitOnly() { return hasWaitOnly(deviceType: mlir::acc::DeviceType::None); }
3961
3962bool DataOp::hasWaitOnly(mlir::acc::DeviceType deviceType) {
3963 return hasDeviceType(arrayAttr: getWaitOnly(), deviceType);
3964}
3965
3966mlir::Operation::operand_range DataOp::getWaitValues() {
3967 return getWaitValues(deviceType: mlir::acc::DeviceType::None);
3968}
3969
3970mlir::Operation::operand_range
3971DataOp::getWaitValues(mlir::acc::DeviceType deviceType) {
3972 return getWaitValuesWithoutDevnum(
3973 deviceTypeAttr: getWaitOperandsDeviceType(), operands: getWaitOperands(), segments: getWaitOperandsSegments(),
3974 hasWaitDevnum: getHasWaitDevnum(), deviceType);
3975}
3976
3977mlir::Value DataOp::getWaitDevnum() {
3978 return getWaitDevnum(deviceType: mlir::acc::DeviceType::None);
3979}
3980
3981mlir::Value DataOp::getWaitDevnum(mlir::acc::DeviceType deviceType) {
3982 return getWaitDevnumValue(deviceTypeAttr: getWaitOperandsDeviceType(), operands: getWaitOperands(),
3983 segments: getWaitOperandsSegments(), hasWaitDevnum: getHasWaitDevnum(),
3984 deviceType);
3985}
3986
3987void acc::DataOp::addAsyncOnly(
3988 MLIRContext *context, llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
3989 setAsyncOnlyAttr(addDeviceTypeAffectedOperandHelper(
3990 context, existingDeviceTypes: getAsyncOnlyAttr(), newDeviceTypes: effectiveDeviceTypes));
3991}
3992
3993void acc::DataOp::addAsyncOperand(
3994 MLIRContext *context, mlir::Value newValue,
3995 llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
3996 setAsyncOperandsDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(
3997 context, existingDeviceTypes: getAsyncOperandsDeviceTypeAttr(), newDeviceTypes: effectiveDeviceTypes, arguments: newValue,
3998 argCollection: getAsyncOperandsMutable()));
3999}
4000
4001void acc::DataOp::addWaitOnly(MLIRContext *context,
4002 llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
4003 setWaitOnlyAttr(addDeviceTypeAffectedOperandHelper(context, existingDeviceTypes: getWaitOnlyAttr(),
4004 newDeviceTypes: effectiveDeviceTypes));
4005}
4006
4007void acc::DataOp::addWaitOperands(
4008 MLIRContext *context, bool hasDevnum, mlir::ValueRange newValues,
4009 llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
4010
4011 llvm::SmallVector<int32_t> segments;
4012 if (getWaitOperandsSegments())
4013 llvm::copy(Range: *getWaitOperandsSegments(), Out: std::back_inserter(x&: segments));
4014
4015 setWaitOperandsDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(
4016 context, existingDeviceTypes: getWaitOperandsDeviceTypeAttr(), newDeviceTypes: effectiveDeviceTypes, arguments: newValues,
4017 argCollection: getWaitOperandsMutable(), segments));
4018 setWaitOperandsSegments(segments);
4019
4020 llvm::SmallVector<mlir::Attribute> hasDevnums;
4021 if (getHasWaitDevnumAttr())
4022 llvm::copy(Range: getHasWaitDevnumAttr(), Out: std::back_inserter(x&: hasDevnums));
4023 hasDevnums.insert(
4024 I: hasDevnums.end(),
4025 NumToInsert: std::max(a: effectiveDeviceTypes.size(), b: static_cast<size_t>(1)),
4026 Elt: mlir::BoolAttr::get(context, value: hasDevnum));
4027 setHasWaitDevnumAttr(mlir::ArrayAttr::get(context, value: hasDevnums));
4028}
4029
4030//===----------------------------------------------------------------------===//
4031// ExitDataOp
4032//===----------------------------------------------------------------------===//
4033
4034LogicalResult acc::ExitDataOp::verify() {
4035 // 2.6.6. Data Exit Directive restriction
4036 // At least one copyout, delete, or detach clause must appear on an exit data
4037 // directive.
4038 if (getDataClauseOperands().empty())
4039 return emitError(message: "at least one operand must be present in dataOperands on "
4040 "the exit data operation");
4041
4042 // The async attribute represent the async clause without value. Therefore the
4043 // attribute and operand cannot appear at the same time.
4044 if (getAsyncOperand() && getAsync())
4045 return emitError(message: "async attribute cannot appear with asyncOperand");
4046
4047 // The wait attribute represent the wait clause without values. Therefore the
4048 // attribute and operands cannot appear at the same time.
4049 if (!getWaitOperands().empty() && getWait())
4050 return emitError(message: "wait attribute cannot appear with waitOperands");
4051
4052 if (getWaitDevnum() && getWaitOperands().empty())
4053 return emitError(message: "wait_devnum cannot appear without waitOperands");
4054
4055 return success();
4056}
4057
4058unsigned ExitDataOp::getNumDataOperands() {
4059 return getDataClauseOperands().size();
4060}
4061
4062Value ExitDataOp::getDataOperand(unsigned i) {
4063 unsigned numOptional = getIfCond() ? 1 : 0;
4064 numOptional += getAsyncOperand() ? 1 : 0;
4065 numOptional += getWaitDevnum() ? 1 : 0;
4066 return getOperand(i: getWaitOperands().size() + numOptional + i);
4067}
4068
4069void ExitDataOp::getCanonicalizationPatterns(RewritePatternSet &results,
4070 MLIRContext *context) {
4071 results.add<RemoveConstantIfCondition<ExitDataOp>>(arg&: context);
4072}
4073
4074void ExitDataOp::addAsyncOnly(MLIRContext *context,
4075 llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
4076 assert(effectiveDeviceTypes.empty());
4077 assert(!getAsyncAttr());
4078 assert(!getAsyncOperand());
4079
4080 setAsyncAttr(mlir::UnitAttr::get(context));
4081}
4082
4083void ExitDataOp::addAsyncOperand(
4084 MLIRContext *context, mlir::Value newValue,
4085 llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
4086 assert(effectiveDeviceTypes.empty());
4087 assert(!getAsyncAttr());
4088 assert(!getAsyncOperand());
4089
4090 getAsyncOperandMutable().append(values: newValue);
4091}
4092
4093void ExitDataOp::addWaitOnly(MLIRContext *context,
4094 llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
4095 assert(effectiveDeviceTypes.empty());
4096
4097 if (getWaitAttr())
4098 return;
4099
4100 setWaitAttr(mlir::UnitAttr::get(context));
4101
4102 getWaitDevnumMutable().clear();
4103 getWaitOperandsMutable().clear();
4104}
4105
4106void ExitDataOp::addWaitOperands(
4107 MLIRContext *context, bool hasDevnum, mlir::ValueRange newValues,
4108 llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
4109 assert(effectiveDeviceTypes.empty());
4110
4111 if (getWaitAttr())
4112 return;
4113
4114 // FIXME: At one point we need to figure out how to support multiple devnums
4115 // here. For now, assert. Eventually we probably want to make dev-num and
4116 // operands work in 'lock-step', so that getWaitDevnum().size() ==
4117 // getWaitOperandsMutable().size().
4118 assert(!getWaitDevnum() && "Merging devnum not yet implemented");
4119
4120 // if hasDevnum, the first value is the devnum. The 'rest' go into the
4121 // operands list.
4122 if (hasDevnum) {
4123 getWaitDevnumMutable().append(values: newValues.front());
4124 newValues = newValues.drop_front();
4125 }
4126
4127 getWaitOperandsMutable().append(values: newValues);
4128}
4129
4130//===----------------------------------------------------------------------===//
4131// EnterDataOp
4132//===----------------------------------------------------------------------===//
4133
4134LogicalResult acc::EnterDataOp::verify() {
4135 // 2.6.6. Data Enter Directive restriction
4136 // At least one copyin, create, or attach clause must appear on an enter data
4137 // directive.
4138 if (getDataClauseOperands().empty())
4139 return emitError(message: "at least one operand must be present in dataOperands on "
4140 "the enter data operation");
4141
4142 // The async attribute represent the async clause without value. Therefore the
4143 // attribute and operand cannot appear at the same time.
4144 if (getAsyncOperand() && getAsync())
4145 return emitError(message: "async attribute cannot appear with asyncOperand");
4146
4147 // The wait attribute represent the wait clause without values. Therefore the
4148 // attribute and operands cannot appear at the same time.
4149 if (!getWaitOperands().empty() && getWait())
4150 return emitError(message: "wait attribute cannot appear with waitOperands");
4151
4152 if (getWaitDevnum() && getWaitOperands().empty())
4153 return emitError(message: "wait_devnum cannot appear without waitOperands");
4154
4155 for (mlir::Value operand : getDataClauseOperands())
4156 if (!mlir::isa<acc::AttachOp, acc::CreateOp, acc::CopyinOp, acc::MapInfoOp>(
4157 Val: operand.getDefiningOp()))
4158 return emitError(message: "expect data entry operation as defining op");
4159
4160 return success();
4161}
4162
4163unsigned EnterDataOp::getNumDataOperands() {
4164 return getDataClauseOperands().size();
4165}
4166
4167Value EnterDataOp::getDataOperand(unsigned i) {
4168 unsigned numOptional = getIfCond() ? 1 : 0;
4169 numOptional += getAsyncOperand() ? 1 : 0;
4170 numOptional += getWaitDevnum() ? 1 : 0;
4171 return getOperand(i: getWaitOperands().size() + numOptional + i);
4172}
4173
4174void EnterDataOp::getCanonicalizationPatterns(RewritePatternSet &results,
4175 MLIRContext *context) {
4176 results.add<RemoveConstantIfCondition<EnterDataOp>>(arg&: context);
4177}
4178
4179void EnterDataOp::addAsyncOnly(
4180 MLIRContext *context, llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
4181 assert(effectiveDeviceTypes.empty());
4182 assert(!getAsyncAttr());
4183 assert(!getAsyncOperand());
4184
4185 setAsyncAttr(mlir::UnitAttr::get(context));
4186}
4187
4188void EnterDataOp::addAsyncOperand(
4189 MLIRContext *context, mlir::Value newValue,
4190 llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
4191 assert(effectiveDeviceTypes.empty());
4192 assert(!getAsyncAttr());
4193 assert(!getAsyncOperand());
4194
4195 getAsyncOperandMutable().append(values: newValue);
4196}
4197
4198void EnterDataOp::addWaitOnly(MLIRContext *context,
4199 llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
4200 assert(effectiveDeviceTypes.empty());
4201
4202 if (getWaitAttr())
4203 return;
4204
4205 setWaitAttr(mlir::UnitAttr::get(context));
4206
4207 getWaitDevnumMutable().clear();
4208 getWaitOperandsMutable().clear();
4209}
4210
4211void EnterDataOp::addWaitOperands(
4212 MLIRContext *context, bool hasDevnum, mlir::ValueRange newValues,
4213 llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
4214 assert(effectiveDeviceTypes.empty());
4215
4216 if (getWaitAttr())
4217 return;
4218
4219 // FIXME: At one point we need to figure out how to support multiple devnums
4220 // here. For now, assert. Eventually we probably want to make dev-num and
4221 // operands work in 'lock-step', so that getWaitDevnum().size() ==
4222 // getWaitOperandsMutable().size().
4223 assert(!getWaitDevnum() && "Merging devnum not yet implemented");
4224
4225 // if hasDevnum, the first value is the devnum. The 'rest' go into the
4226 // operands list.
4227 if (hasDevnum) {
4228 getWaitDevnumMutable().append(values: newValues.front());
4229 newValues = newValues.drop_front();
4230 }
4231
4232 getWaitOperandsMutable().append(values: newValues);
4233}
4234
4235//===----------------------------------------------------------------------===//
4236// AtomicReadOp
4237//===----------------------------------------------------------------------===//
4238
4239LogicalResult AtomicReadOp::verify() { return verifyCommon(); }
4240
4241//===----------------------------------------------------------------------===//
4242// AtomicWriteOp
4243//===----------------------------------------------------------------------===//
4244
4245LogicalResult AtomicWriteOp::verify() { return verifyCommon(); }
4246
4247//===----------------------------------------------------------------------===//
4248// AtomicUpdateOp
4249//===----------------------------------------------------------------------===//
4250
4251LogicalResult AtomicUpdateOp::canonicalize(AtomicUpdateOp op,
4252 PatternRewriter &rewriter) {
4253 if (op.isNoOp()) {
4254 rewriter.eraseOp(op);
4255 return success();
4256 }
4257
4258 if (Value writeVal = op.getWriteOpVal()) {
4259 rewriter.replaceOpWithNewOp<AtomicWriteOp>(op, args: op.getX(), args&: writeVal,
4260 args: op.getIfCond());
4261 return success();
4262 }
4263
4264 return failure();
4265}
4266
4267LogicalResult AtomicUpdateOp::verify() { return verifyCommon(); }
4268
4269LogicalResult AtomicUpdateOp::verifyRegions() { return verifyRegionsCommon(); }
4270
4271//===----------------------------------------------------------------------===//
4272// AtomicCaptureOp
4273//===----------------------------------------------------------------------===//
4274
4275AtomicReadOp AtomicCaptureOp::getAtomicReadOp() {
4276 if (auto op = dyn_cast<AtomicReadOp>(Val: getFirstOp()))
4277 return op;
4278 return dyn_cast<AtomicReadOp>(Val: getSecondOp());
4279}
4280
4281AtomicWriteOp AtomicCaptureOp::getAtomicWriteOp() {
4282 if (auto op = dyn_cast<AtomicWriteOp>(Val: getFirstOp()))
4283 return op;
4284 return dyn_cast<AtomicWriteOp>(Val: getSecondOp());
4285}
4286
4287AtomicUpdateOp AtomicCaptureOp::getAtomicUpdateOp() {
4288 if (auto op = dyn_cast<AtomicUpdateOp>(Val: getFirstOp()))
4289 return op;
4290 return dyn_cast<AtomicUpdateOp>(Val: getSecondOp());
4291}
4292
4293LogicalResult AtomicCaptureOp::verifyRegions() { return verifyRegionsCommon(); }
4294
4295//===----------------------------------------------------------------------===//
4296// DeclareEnterOp
4297//===----------------------------------------------------------------------===//
4298
4299template <typename Op>
4300static LogicalResult
4301checkDeclareOperands(Op &op, const mlir::ValueRange &operands,
4302 bool requireAtLeastOneOperand = true) {
4303 if (operands.empty() && requireAtLeastOneOperand)
4304 return emitError(
4305 op->getLoc(),
4306 "at least one operand must appear on the declare operation");
4307
4308 for (mlir::Value operand : operands) {
4309 if (isa<BlockArgument>(Val: operand) ||
4310 !mlir::isa<acc::CopyinOp, acc::CopyoutOp, acc::CreateOp,
4311 acc::DevicePtrOp, acc::GetDevicePtrOp, acc::PresentOp,
4312 acc::DeclareDeviceResidentOp, acc::DeclareLinkOp,
4313 acc::MapInfoOp>(Val: operand.getDefiningOp()))
4314 return op.emitError(
4315 "expect valid declare data entry operation or acc.getdeviceptr "
4316 "as defining op");
4317
4318 mlir::Value var{getVar(accDataClauseOp: operand.getDefiningOp())};
4319 assert(var && "declare operands can only be data entry operations which "
4320 "must have var");
4321 (void)var;
4322 // acc.map_info encodes the clause effects in mapFlags instead.
4323 if (!mlir::isa<acc::MapInfoOp>(Val: operand.getDefiningOp())) {
4324 std::optional<mlir::acc::DataClause> dataClauseOptional{
4325 getDataClause(accDataEntryOp: operand.getDefiningOp())};
4326 assert(dataClauseOptional.has_value() &&
4327 "declare operands can only be data entry operations which must "
4328 "have dataClause");
4329 (void)dataClauseOptional;
4330 }
4331 }
4332
4333 return success();
4334}
4335
4336LogicalResult acc::DeclareEnterOp::verify() {
4337 return checkDeclareOperands(op&: *this, operands: this->getDataClauseOperands());
4338}
4339
4340//===----------------------------------------------------------------------===//
4341// DeclareExitOp
4342//===----------------------------------------------------------------------===//
4343
4344LogicalResult acc::DeclareExitOp::verify() {
4345 if (getToken())
4346 return checkDeclareOperands(op&: *this, operands: this->getDataClauseOperands(),
4347 /*requireAtLeastOneOperand=*/false);
4348 return checkDeclareOperands(op&: *this, operands: this->getDataClauseOperands());
4349}
4350
4351//===----------------------------------------------------------------------===//
4352// DeclareOp
4353//===----------------------------------------------------------------------===//
4354
4355LogicalResult acc::DeclareOp::verify() {
4356 return checkDeclareOperands(op&: *this, operands: this->getDataClauseOperands());
4357}
4358
4359//===----------------------------------------------------------------------===//
4360// RoutineOp
4361//===----------------------------------------------------------------------===//
4362
4363static unsigned getParallelismForDeviceType(acc::RoutineOp op,
4364 acc::DeviceType dtype) {
4365 unsigned parallelism = 0;
4366 parallelism += (op.hasGang(deviceType: dtype) || op.getGangDimValue(deviceType: dtype)) ? 1 : 0;
4367 parallelism += op.hasWorker(deviceType: dtype) ? 1 : 0;
4368 parallelism += op.hasVector(deviceType: dtype) ? 1 : 0;
4369 parallelism += op.hasSeq(deviceType: dtype) ? 1 : 0;
4370 return parallelism;
4371}
4372
4373LogicalResult acc::RoutineOp::verify() {
4374 unsigned baseParallelism =
4375 getParallelismForDeviceType(op: *this, dtype: acc::DeviceType::None);
4376
4377 if (baseParallelism > 1)
4378 return emitError() << "only one of `gang`, `worker`, `vector`, `seq` can "
4379 "be present at the same time";
4380
4381 for (uint32_t dtypeInt = 0; dtypeInt != acc::getMaxEnumValForDeviceType();
4382 ++dtypeInt) {
4383 auto dtype = static_cast<acc::DeviceType>(dtypeInt);
4384 if (dtype == acc::DeviceType::None)
4385 continue;
4386 unsigned parallelism = getParallelismForDeviceType(op: *this, dtype);
4387
4388 if (parallelism > 1 || (baseParallelism == 1 && parallelism == 1))
4389 return emitError() << "only one of `gang`, `worker`, `vector`, `seq` can "
4390 "be present at the same time for device_type `"
4391 << acc::stringifyDeviceType(val: dtype) << "`";
4392 }
4393
4394 return success();
4395}
4396
4397static ParseResult parseBindName(OpAsmParser &parser,
4398 mlir::ArrayAttr &bindIdName,
4399 mlir::ArrayAttr &bindStrName,
4400 mlir::ArrayAttr &deviceIdTypes,
4401 mlir::ArrayAttr &deviceStrTypes) {
4402 llvm::SmallVector<mlir::Attribute> bindIdNameAttrs;
4403 llvm::SmallVector<mlir::Attribute> bindStrNameAttrs;
4404 llvm::SmallVector<mlir::Attribute> deviceIdTypeAttrs;
4405 llvm::SmallVector<mlir::Attribute> deviceStrTypeAttrs;
4406
4407 if (failed(Result: parser.parseCommaSeparatedList(parseElementFn: [&]() {
4408 llvm::SMLoc attrLoc = parser.getCurrentLocation();
4409 mlir::Attribute newAttr;
4410 bool isSymbolRefAttr;
4411 if (parser.parseAttribute(result&: newAttr))
4412 return failure();
4413 if (auto symbolRefAttr = dyn_cast<mlir::SymbolRefAttr>(Val&: newAttr)) {
4414 bindIdNameAttrs.push_back(Elt: symbolRefAttr);
4415 isSymbolRefAttr = true;
4416 } else if (auto stringAttr = dyn_cast<mlir::StringAttr>(Val&: newAttr)) {
4417 bindStrNameAttrs.push_back(Elt: stringAttr);
4418 isSymbolRefAttr = false;
4419 } else {
4420 parser.emitError(loc: attrLoc,
4421 message: "expected symbol reference or string attribute");
4422 return failure();
4423 }
4424 if (failed(Result: parser.parseOptionalLSquare())) {
4425 if (isSymbolRefAttr) {
4426 deviceIdTypeAttrs.push_back(Elt: mlir::acc::DeviceTypeAttr::get(
4427 context: parser.getContext(), value: mlir::acc::DeviceType::None));
4428 } else {
4429 deviceStrTypeAttrs.push_back(Elt: mlir::acc::DeviceTypeAttr::get(
4430 context: parser.getContext(), value: mlir::acc::DeviceType::None));
4431 }
4432 } else {
4433 if (isSymbolRefAttr) {
4434 if (parser.parseAttribute(result&: deviceIdTypeAttrs.emplace_back()) ||
4435 parser.parseRSquare())
4436 return failure();
4437 } else {
4438 if (parser.parseAttribute(result&: deviceStrTypeAttrs.emplace_back()) ||
4439 parser.parseRSquare())
4440 return failure();
4441 }
4442 }
4443 return success();
4444 })))
4445 return failure();
4446
4447 bindIdName = ArrayAttr::get(context: parser.getContext(), value: bindIdNameAttrs);
4448 bindStrName = ArrayAttr::get(context: parser.getContext(), value: bindStrNameAttrs);
4449 deviceIdTypes = ArrayAttr::get(context: parser.getContext(), value: deviceIdTypeAttrs);
4450 deviceStrTypes = ArrayAttr::get(context: parser.getContext(), value: deviceStrTypeAttrs);
4451
4452 return success();
4453}
4454
4455static void printBindName(mlir::OpAsmPrinter &p, mlir::Operation *op,
4456 std::optional<mlir::ArrayAttr> bindIdName,
4457 std::optional<mlir::ArrayAttr> bindStrName,
4458 std::optional<mlir::ArrayAttr> deviceIdTypes,
4459 std::optional<mlir::ArrayAttr> deviceStrTypes) {
4460 // Create combined vectors for all bind names and device types
4461 llvm::SmallVector<mlir::Attribute> allBindNames;
4462 llvm::SmallVector<mlir::Attribute> allDeviceTypes;
4463
4464 // Append bindIdName and deviceIdTypes
4465 if (hasDeviceTypeValues(arrayAttr: deviceIdTypes)) {
4466 allBindNames.append(in_start: bindIdName->begin(), in_end: bindIdName->end());
4467 allDeviceTypes.append(in_start: deviceIdTypes->begin(), in_end: deviceIdTypes->end());
4468 }
4469
4470 // Append bindStrName and deviceStrTypes
4471 if (hasDeviceTypeValues(arrayAttr: deviceStrTypes)) {
4472 allBindNames.append(in_start: bindStrName->begin(), in_end: bindStrName->end());
4473 allDeviceTypes.append(in_start: deviceStrTypes->begin(), in_end: deviceStrTypes->end());
4474 }
4475
4476 // Print the combined sequence
4477 if (!allBindNames.empty())
4478 llvm::interleaveComma(c: llvm::zip(t&: allBindNames, u&: allDeviceTypes), os&: p,
4479 each_fn: [&](const auto &pair) {
4480 p << std::get<0>(pair);
4481 printSingleDeviceType(p, std::get<1>(pair));
4482 });
4483}
4484
4485static ParseResult parseRoutineGangClause(OpAsmParser &parser,
4486 mlir::ArrayAttr &gang,
4487 mlir::ArrayAttr &gangDim,
4488 mlir::ArrayAttr &gangDimDeviceTypes) {
4489
4490 llvm::SmallVector<mlir::Attribute> gangAttrs, gangDimAttrs,
4491 gangDimDeviceTypeAttrs;
4492 bool needCommaBeforeOperands = false;
4493
4494 // Gang keyword only
4495 if (failed(Result: parser.parseOptionalLParen())) {
4496 gangAttrs.push_back(Elt: mlir::acc::DeviceTypeAttr::get(
4497 context: parser.getContext(), value: mlir::acc::DeviceType::None));
4498 gang = ArrayAttr::get(context: parser.getContext(), value: gangAttrs);
4499 return success();
4500 }
4501
4502 // Parse keyword only attributes
4503 if (succeeded(Result: parser.parseOptionalLSquare())) {
4504 if (failed(Result: parser.parseCommaSeparatedList(parseElementFn: [&]() {
4505 if (parser.parseAttribute(result&: gangAttrs.emplace_back()))
4506 return failure();
4507 return success();
4508 })))
4509 return failure();
4510 if (parser.parseRSquare())
4511 return failure();
4512 needCommaBeforeOperands = true;
4513 }
4514
4515 if (needCommaBeforeOperands && failed(Result: parser.parseComma()))
4516 return failure();
4517
4518 if (failed(Result: parser.parseCommaSeparatedList(parseElementFn: [&]() {
4519 if (parser.parseKeyword(keyword: acc::RoutineOp::getGangDimKeyword()) ||
4520 parser.parseColon() ||
4521 parser.parseAttribute(result&: gangDimAttrs.emplace_back()))
4522 return failure();
4523 if (succeeded(Result: parser.parseOptionalLSquare())) {
4524 if (parser.parseAttribute(result&: gangDimDeviceTypeAttrs.emplace_back()) ||
4525 parser.parseRSquare())
4526 return failure();
4527 } else {
4528 gangDimDeviceTypeAttrs.push_back(Elt: mlir::acc::DeviceTypeAttr::get(
4529 context: parser.getContext(), value: mlir::acc::DeviceType::None));
4530 }
4531 return success();
4532 })))
4533 return failure();
4534
4535 if (failed(Result: parser.parseRParen()))
4536 return failure();
4537
4538 gang = ArrayAttr::get(context: parser.getContext(), value: gangAttrs);
4539 gangDim = ArrayAttr::get(context: parser.getContext(), value: gangDimAttrs);
4540 gangDimDeviceTypes =
4541 ArrayAttr::get(context: parser.getContext(), value: gangDimDeviceTypeAttrs);
4542
4543 return success();
4544}
4545
4546void printRoutineGangClause(OpAsmPrinter &p, Operation *op,
4547 std::optional<mlir::ArrayAttr> gang,
4548 std::optional<mlir::ArrayAttr> gangDim,
4549 std::optional<mlir::ArrayAttr> gangDimDeviceTypes) {
4550
4551 if (!hasDeviceTypeValues(arrayAttr: gangDimDeviceTypes) && hasDeviceTypeValues(arrayAttr: gang) &&
4552 gang->size() == 1) {
4553 auto deviceTypeAttr = mlir::dyn_cast<mlir::acc::DeviceTypeAttr>(Val: (*gang)[0]);
4554 if (deviceTypeAttr.getValue() == mlir::acc::DeviceType::None)
4555 return;
4556 }
4557
4558 p << "(";
4559
4560 printDeviceTypes(p, deviceTypes: gang);
4561
4562 if (hasDeviceTypeValues(arrayAttr: gang) && hasDeviceTypeValues(arrayAttr: gangDimDeviceTypes))
4563 p << ", ";
4564
4565 if (hasDeviceTypeValues(arrayAttr: gangDimDeviceTypes))
4566 llvm::interleaveComma(c: llvm::zip(t&: *gangDim, u&: *gangDimDeviceTypes), os&: p,
4567 each_fn: [&](const auto &pair) {
4568 p << acc::RoutineOp::getGangDimKeyword() << ": ";
4569 p << std::get<0>(pair);
4570 printSingleDeviceType(p, std::get<1>(pair));
4571 });
4572
4573 p << ")";
4574}
4575
4576static ParseResult parseDeviceTypeArrayAttr(OpAsmParser &parser,
4577 mlir::ArrayAttr &deviceTypes) {
4578 llvm::SmallVector<mlir::Attribute> attributes;
4579 // Keyword only
4580 if (failed(Result: parser.parseOptionalLParen())) {
4581 attributes.push_back(Elt: mlir::acc::DeviceTypeAttr::get(
4582 context: parser.getContext(), value: mlir::acc::DeviceType::None));
4583 deviceTypes = ArrayAttr::get(context: parser.getContext(), value: attributes);
4584 return success();
4585 }
4586
4587 // Parse device type attributes
4588 if (succeeded(Result: parser.parseOptionalLSquare())) {
4589 if (failed(Result: parser.parseCommaSeparatedList(parseElementFn: [&]() {
4590 if (parser.parseAttribute(result&: attributes.emplace_back()))
4591 return failure();
4592 return success();
4593 })))
4594 return failure();
4595 if (parser.parseRSquare() || parser.parseRParen())
4596 return failure();
4597 }
4598 deviceTypes = ArrayAttr::get(context: parser.getContext(), value: attributes);
4599 return success();
4600}
4601
4602static void
4603printDeviceTypeArrayAttr(mlir::OpAsmPrinter &p, mlir::Operation *op,
4604 std::optional<mlir::ArrayAttr> deviceTypes) {
4605
4606 if (hasDeviceTypeValues(arrayAttr: deviceTypes) && deviceTypes->size() == 1) {
4607 auto deviceTypeAttr =
4608 mlir::dyn_cast<mlir::acc::DeviceTypeAttr>(Val: (*deviceTypes)[0]);
4609 if (deviceTypeAttr.getValue() == mlir::acc::DeviceType::None)
4610 return;
4611 }
4612
4613 if (!hasDeviceTypeValues(arrayAttr: deviceTypes))
4614 return;
4615
4616 p << "([";
4617 llvm::interleaveComma(c: *deviceTypes, os&: p, each_fn: [&](mlir::Attribute attr) {
4618 auto dTypeAttr = mlir::dyn_cast<mlir::acc::DeviceTypeAttr>(Val&: attr);
4619 p << dTypeAttr;
4620 });
4621 p << "])";
4622}
4623
4624bool RoutineOp::hasWorker() { return hasWorker(deviceType: mlir::acc::DeviceType::None); }
4625
4626bool RoutineOp::hasWorker(mlir::acc::DeviceType deviceType) {
4627 return hasDeviceType(arrayAttr: getWorker(), deviceType);
4628}
4629
4630bool RoutineOp::hasVector() { return hasVector(deviceType: mlir::acc::DeviceType::None); }
4631
4632bool RoutineOp::hasVector(mlir::acc::DeviceType deviceType) {
4633 return hasDeviceType(arrayAttr: getVector(), deviceType);
4634}
4635
4636bool RoutineOp::hasSeq() { return hasSeq(deviceType: mlir::acc::DeviceType::None); }
4637
4638bool RoutineOp::hasSeq(mlir::acc::DeviceType deviceType) {
4639 return hasDeviceType(arrayAttr: getSeq(), deviceType);
4640}
4641
4642std::optional<std::variant<mlir::SymbolRefAttr, mlir::StringAttr>>
4643RoutineOp::getBindNameValue() {
4644 return getBindNameValue(deviceType: mlir::acc::DeviceType::None);
4645}
4646
4647std::optional<std::variant<mlir::SymbolRefAttr, mlir::StringAttr>>
4648RoutineOp::getBindNameValue(mlir::acc::DeviceType deviceType) {
4649 if (hasDeviceTypeValues(arrayAttr: getBindIdNameDeviceType())) {
4650 if (auto pos = findSegment(segments: *getBindIdNameDeviceType(), deviceType)) {
4651 auto attr = (*getBindIdName())[*pos];
4652 auto symbolRefAttr = dyn_cast<mlir::SymbolRefAttr>(Val&: attr);
4653 assert(symbolRefAttr && "expected SymbolRef");
4654 return symbolRefAttr;
4655 }
4656 }
4657
4658 if (hasDeviceTypeValues(arrayAttr: getBindStrNameDeviceType())) {
4659 if (auto pos = findSegment(segments: *getBindStrNameDeviceType(), deviceType)) {
4660 auto attr = (*getBindStrName())[*pos];
4661 auto stringAttr = dyn_cast<mlir::StringAttr>(Val&: attr);
4662 assert(stringAttr && "expected String");
4663 return stringAttr;
4664 }
4665 }
4666
4667 return std::nullopt;
4668}
4669
4670std::optional<std::variant<mlir::SymbolRefAttr, mlir::StringAttr>>
4671RoutineOp::getEffectiveBindName(mlir::acc::DeviceType targetDeviceType) {
4672 assert(targetDeviceType != DeviceType::Star &&
4673 "expected a valid target device type");
4674
4675 if (!getBindIdName() && !getBindStrName())
4676 return std::nullopt;
4677
4678 // None queries the default bind.
4679 if (targetDeviceType == DeviceType::None)
4680 return getBindNameValue();
4681
4682 // Prefer an exact device-specific bind.
4683 if (auto bindName = getBindNameValue(deviceType: targetDeviceType))
4684 return bindName;
4685
4686 // The device_type(*) clause applies only if no other device_type clause names
4687 // the target device.
4688 bool hasNamedDeviceType = hasGang(deviceType: targetDeviceType) ||
4689 getGangDimValue(deviceType: targetDeviceType).has_value() ||
4690 hasWorker(deviceType: targetDeviceType) ||
4691 hasVector(deviceType: targetDeviceType) ||
4692 hasSeq(deviceType: targetDeviceType);
4693 if (!hasNamedDeviceType) {
4694 if (auto bindName = getBindNameValue(deviceType: DeviceType::Star))
4695 return bindName;
4696 }
4697
4698 return getBindNameValue();
4699}
4700
4701bool RoutineOp::hasGang() { return hasGang(deviceType: mlir::acc::DeviceType::None); }
4702
4703bool RoutineOp::hasGang(mlir::acc::DeviceType deviceType) {
4704 return hasDeviceType(arrayAttr: getGang(), deviceType);
4705}
4706
4707std::optional<int64_t> RoutineOp::getGangDimValue() {
4708 return getGangDimValue(deviceType: mlir::acc::DeviceType::None);
4709}
4710
4711std::optional<int64_t>
4712RoutineOp::getGangDimValue(mlir::acc::DeviceType deviceType) {
4713 if (!hasDeviceTypeValues(arrayAttr: getGangDimDeviceType()))
4714 return std::nullopt;
4715 if (auto pos = findSegment(segments: *getGangDimDeviceType(), deviceType)) {
4716 auto intAttr = mlir::dyn_cast<mlir::IntegerAttr>(Val: (*getGangDim())[*pos]);
4717 return intAttr.getInt();
4718 }
4719 return std::nullopt;
4720}
4721
4722void RoutineOp::addSeq(MLIRContext *context,
4723 llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
4724 setSeqAttr(addDeviceTypeAffectedOperandHelper(context, existingDeviceTypes: getSeqAttr(),
4725 newDeviceTypes: effectiveDeviceTypes));
4726}
4727
4728void RoutineOp::addVector(MLIRContext *context,
4729 llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
4730 setVectorAttr(addDeviceTypeAffectedOperandHelper(context, existingDeviceTypes: getVectorAttr(),
4731 newDeviceTypes: effectiveDeviceTypes));
4732}
4733
4734void RoutineOp::addWorker(MLIRContext *context,
4735 llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
4736 setWorkerAttr(addDeviceTypeAffectedOperandHelper(context, existingDeviceTypes: getWorkerAttr(),
4737 newDeviceTypes: effectiveDeviceTypes));
4738}
4739
4740void RoutineOp::addGang(MLIRContext *context,
4741 llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
4742 setGangAttr(addDeviceTypeAffectedOperandHelper(context, existingDeviceTypes: getGangAttr(),
4743 newDeviceTypes: effectiveDeviceTypes));
4744}
4745
4746void RoutineOp::addGang(MLIRContext *context,
4747 llvm::ArrayRef<DeviceType> effectiveDeviceTypes,
4748 uint64_t val) {
4749 llvm::SmallVector<mlir::Attribute> dimValues;
4750 llvm::SmallVector<mlir::Attribute> deviceTypes;
4751
4752 if (getGangDimAttr())
4753 llvm::copy(Range: getGangDimAttr(), Out: std::back_inserter(x&: dimValues));
4754 if (getGangDimDeviceTypeAttr())
4755 llvm::copy(Range: getGangDimDeviceTypeAttr(), Out: std::back_inserter(x&: deviceTypes));
4756
4757 assert(dimValues.size() == deviceTypes.size());
4758
4759 if (effectiveDeviceTypes.empty()) {
4760 dimValues.push_back(
4761 Elt: mlir::IntegerAttr::get(type: mlir::IntegerType::get(context, width: 64), value: val));
4762 deviceTypes.push_back(
4763 Elt: acc::DeviceTypeAttr::get(context, value: acc::DeviceType::None));
4764 } else {
4765 for (DeviceType dt : effectiveDeviceTypes) {
4766 dimValues.push_back(
4767 Elt: mlir::IntegerAttr::get(type: mlir::IntegerType::get(context, width: 64), value: val));
4768 deviceTypes.push_back(Elt: acc::DeviceTypeAttr::get(context, value: dt));
4769 }
4770 }
4771 assert(dimValues.size() == deviceTypes.size());
4772
4773 setGangDimAttr(mlir::ArrayAttr::get(context, value: dimValues));
4774 setGangDimDeviceTypeAttr(mlir::ArrayAttr::get(context, value: deviceTypes));
4775}
4776
4777void RoutineOp::addBindStrName(MLIRContext *context,
4778 llvm::ArrayRef<DeviceType> effectiveDeviceTypes,
4779 mlir::StringAttr val) {
4780 unsigned before = getBindStrNameDeviceTypeAttr()
4781 ? getBindStrNameDeviceTypeAttr().size()
4782 : 0;
4783
4784 setBindStrNameDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(
4785 context, existingDeviceTypes: getBindStrNameDeviceTypeAttr(), newDeviceTypes: effectiveDeviceTypes));
4786 unsigned after = getBindStrNameDeviceTypeAttr().size();
4787
4788 llvm::SmallVector<mlir::Attribute> vals;
4789 if (getBindStrNameAttr())
4790 llvm::copy(Range: getBindStrNameAttr(), Out: std::back_inserter(x&: vals));
4791 for (unsigned i = 0; i < after - before; ++i)
4792 vals.push_back(Elt: val);
4793
4794 setBindStrNameAttr(mlir::ArrayAttr::get(context, value: vals));
4795}
4796
4797void RoutineOp::addBindIDName(MLIRContext *context,
4798 llvm::ArrayRef<DeviceType> effectiveDeviceTypes,
4799 mlir::SymbolRefAttr val) {
4800 unsigned before =
4801 getBindIdNameDeviceTypeAttr() ? getBindIdNameDeviceTypeAttr().size() : 0;
4802
4803 setBindIdNameDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(
4804 context, existingDeviceTypes: getBindIdNameDeviceTypeAttr(), newDeviceTypes: effectiveDeviceTypes));
4805 unsigned after = getBindIdNameDeviceTypeAttr().size();
4806
4807 llvm::SmallVector<mlir::Attribute> vals;
4808 if (getBindIdNameAttr())
4809 llvm::copy(Range: getBindIdNameAttr(), Out: std::back_inserter(x&: vals));
4810 for (unsigned i = 0; i < after - before; ++i)
4811 vals.push_back(Elt: val);
4812
4813 setBindIdNameAttr(mlir::ArrayAttr::get(context, value: vals));
4814}
4815
4816//===----------------------------------------------------------------------===//
4817// InitOp
4818//===----------------------------------------------------------------------===//
4819
4820LogicalResult acc::InitOp::verify() {
4821 if (getOperation()->getParentOfType<ACC_COMPUTE_CONSTRUCT_AND_LOOP_OPS>())
4822 return emitOpError(message: "cannot be nested in a compute operation");
4823 return success();
4824}
4825
4826void acc::InitOp::addDeviceType(MLIRContext *context,
4827 mlir::acc::DeviceType deviceType) {
4828 llvm::SmallVector<mlir::Attribute> deviceTypes;
4829 if (getDeviceTypesAttr())
4830 llvm::copy(Range: getDeviceTypesAttr(), Out: std::back_inserter(x&: deviceTypes));
4831
4832 deviceTypes.push_back(Elt: acc::DeviceTypeAttr::get(context, value: deviceType));
4833 setDeviceTypesAttr(mlir::ArrayAttr::get(context, value: deviceTypes));
4834}
4835
4836//===----------------------------------------------------------------------===//
4837// ShutdownOp
4838//===----------------------------------------------------------------------===//
4839
4840LogicalResult acc::ShutdownOp::verify() {
4841 if (getOperation()->getParentOfType<ACC_COMPUTE_CONSTRUCT_AND_LOOP_OPS>())
4842 return emitOpError(message: "cannot be nested in a compute operation");
4843 return success();
4844}
4845
4846void acc::ShutdownOp::addDeviceType(MLIRContext *context,
4847 mlir::acc::DeviceType deviceType) {
4848 llvm::SmallVector<mlir::Attribute> deviceTypes;
4849 if (getDeviceTypesAttr())
4850 llvm::copy(Range: getDeviceTypesAttr(), Out: std::back_inserter(x&: deviceTypes));
4851
4852 deviceTypes.push_back(Elt: acc::DeviceTypeAttr::get(context, value: deviceType));
4853 setDeviceTypesAttr(mlir::ArrayAttr::get(context, value: deviceTypes));
4854}
4855
4856//===----------------------------------------------------------------------===//
4857// SetOp
4858//===----------------------------------------------------------------------===//
4859
4860LogicalResult acc::SetOp::verify() {
4861 if (getOperation()->getParentOfType<ACC_COMPUTE_CONSTRUCT_AND_LOOP_OPS>())
4862 return emitOpError(message: "cannot be nested in a compute operation");
4863 if (!getDeviceTypeAttr() && !getDefaultAsync() && !getDeviceNum())
4864 return emitOpError(message: "at least one default_async, device_num, or device_type "
4865 "operand must appear");
4866 return success();
4867}
4868
4869//===----------------------------------------------------------------------===//
4870// UpdateOp
4871//===----------------------------------------------------------------------===//
4872
4873LogicalResult acc::UpdateOp::verify() {
4874 // At least one of host or device should have a value.
4875 if (getDataClauseOperands().empty())
4876 return emitError(message: "at least one value must be present in dataOperands");
4877
4878 if (failed(Result: verifyDeviceTypeCountMatch(op: *this, operands: getAsyncOperands(),
4879 deviceTypes: getAsyncOperandsDeviceTypeAttr(),
4880 keyword: "async")))
4881 return failure();
4882
4883 if (failed(Result: verifyDeviceTypeAndSegmentCountMatch(
4884 op: *this, operands: getWaitOperands(), segments: getWaitOperandsSegmentsAttr(),
4885 deviceTypes: getWaitOperandsDeviceTypeAttr(), keyword: "wait")))
4886 return failure();
4887
4888 if (failed(Result: checkWaitAndAsyncConflict<acc::UpdateOp>(op: *this)))
4889 return failure();
4890
4891 for (mlir::Value operand : getDataClauseOperands())
4892 if (!mlir::isa<acc::UpdateDeviceOp, acc::UpdateHostOp, acc::GetDevicePtrOp,
4893 acc::MapInfoOp>(Val: operand.getDefiningOp()))
4894 return emitError(message: "expect data entry/exit operation or acc.getdeviceptr "
4895 "as defining op");
4896
4897 return success();
4898}
4899
4900unsigned UpdateOp::getNumDataOperands() {
4901 return getDataClauseOperands().size();
4902}
4903
4904Value UpdateOp::getDataOperand(unsigned i) {
4905 unsigned numOptional = getAsyncOperands().size();
4906 numOptional += getIfCond() ? 1 : 0;
4907 return getOperand(i: getWaitOperands().size() + numOptional + i);
4908}
4909
4910void UpdateOp::getCanonicalizationPatterns(RewritePatternSet &results,
4911 MLIRContext *context) {
4912 results.add<RemoveConstantIfCondition<UpdateOp>>(arg&: context);
4913}
4914
4915bool UpdateOp::hasAsyncOnly() {
4916 return hasAsyncOnly(deviceType: mlir::acc::DeviceType::None);
4917}
4918
4919bool UpdateOp::hasAsyncOnly(mlir::acc::DeviceType deviceType) {
4920 return hasDeviceType(arrayAttr: getAsyncOnly(), deviceType);
4921}
4922
4923mlir::Value UpdateOp::getAsyncValue() {
4924 return getAsyncValue(deviceType: mlir::acc::DeviceType::None);
4925}
4926
4927mlir::Value UpdateOp::getAsyncValue(mlir::acc::DeviceType deviceType) {
4928 if (!hasDeviceTypeValues(arrayAttr: getAsyncOperandsDeviceType()))
4929 return {};
4930
4931 if (auto pos = findSegment(segments: *getAsyncOperandsDeviceType(), deviceType))
4932 return getAsyncOperands()[*pos];
4933
4934 return {};
4935}
4936
4937bool UpdateOp::hasWaitOnly() {
4938 return hasWaitOnly(deviceType: mlir::acc::DeviceType::None);
4939}
4940
4941bool UpdateOp::hasWaitOnly(mlir::acc::DeviceType deviceType) {
4942 return hasDeviceType(arrayAttr: getWaitOnly(), deviceType);
4943}
4944
4945mlir::Operation::operand_range UpdateOp::getWaitValues() {
4946 return getWaitValues(deviceType: mlir::acc::DeviceType::None);
4947}
4948
4949mlir::Operation::operand_range
4950UpdateOp::getWaitValues(mlir::acc::DeviceType deviceType) {
4951 return getWaitValuesWithoutDevnum(
4952 deviceTypeAttr: getWaitOperandsDeviceType(), operands: getWaitOperands(), segments: getWaitOperandsSegments(),
4953 hasWaitDevnum: getHasWaitDevnum(), deviceType);
4954}
4955
4956mlir::Value UpdateOp::getWaitDevnum() {
4957 return getWaitDevnum(deviceType: mlir::acc::DeviceType::None);
4958}
4959
4960mlir::Value UpdateOp::getWaitDevnum(mlir::acc::DeviceType deviceType) {
4961 return getWaitDevnumValue(deviceTypeAttr: getWaitOperandsDeviceType(), operands: getWaitOperands(),
4962 segments: getWaitOperandsSegments(), hasWaitDevnum: getHasWaitDevnum(),
4963 deviceType);
4964}
4965
4966void UpdateOp::addAsyncOnly(MLIRContext *context,
4967 llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
4968 setAsyncOnlyAttr(addDeviceTypeAffectedOperandHelper(
4969 context, existingDeviceTypes: getAsyncOnlyAttr(), newDeviceTypes: effectiveDeviceTypes));
4970}
4971
4972void UpdateOp::addAsyncOperand(
4973 MLIRContext *context, mlir::Value newValue,
4974 llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
4975 setAsyncOperandsDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(
4976 context, existingDeviceTypes: getAsyncOperandsDeviceTypeAttr(), newDeviceTypes: effectiveDeviceTypes, arguments: newValue,
4977 argCollection: getAsyncOperandsMutable()));
4978}
4979
4980void UpdateOp::addWaitOnly(MLIRContext *context,
4981 llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
4982 setWaitOnlyAttr(addDeviceTypeAffectedOperandHelper(context, existingDeviceTypes: getWaitOnlyAttr(),
4983 newDeviceTypes: effectiveDeviceTypes));
4984}
4985
4986void UpdateOp::addWaitOperands(
4987 MLIRContext *context, bool hasDevnum, mlir::ValueRange newValues,
4988 llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
4989
4990 llvm::SmallVector<int32_t> segments;
4991 if (getWaitOperandsSegments())
4992 llvm::copy(Range: *getWaitOperandsSegments(), Out: std::back_inserter(x&: segments));
4993
4994 setWaitOperandsDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(
4995 context, existingDeviceTypes: getWaitOperandsDeviceTypeAttr(), newDeviceTypes: effectiveDeviceTypes, arguments: newValues,
4996 argCollection: getWaitOperandsMutable(), segments));
4997 setWaitOperandsSegments(segments);
4998
4999 llvm::SmallVector<mlir::Attribute> hasDevnums;
5000 if (getHasWaitDevnumAttr())
5001 llvm::copy(Range: getHasWaitDevnumAttr(), Out: std::back_inserter(x&: hasDevnums));
5002 hasDevnums.insert(
5003 I: hasDevnums.end(),
5004 NumToInsert: std::max(a: effectiveDeviceTypes.size(), b: static_cast<size_t>(1)),
5005 Elt: mlir::BoolAttr::get(context, value: hasDevnum));
5006 setHasWaitDevnumAttr(mlir::ArrayAttr::get(context, value: hasDevnums));
5007}
5008
5009//===----------------------------------------------------------------------===//
5010// WaitOp
5011//===----------------------------------------------------------------------===//
5012
5013LogicalResult acc::WaitOp::verify() {
5014 // The async attribute represent the async clause without value. Therefore the
5015 // attribute and operand cannot appear at the same time.
5016 if (getAsyncOperand() && getAsync())
5017 return emitError(message: "async attribute cannot appear with asyncOperand");
5018
5019 if (getWaitDevnum() && getWaitOperands().empty())
5020 return emitError(message: "wait_devnum cannot appear without waitOperands");
5021
5022 return success();
5023}
5024
5025#define GET_OP_CLASSES
5026#include "mlir/Dialect/OpenACC/OpenACCOps.cpp.inc"
5027
5028#define GET_ATTRDEF_CLASSES
5029#include "mlir/Dialect/OpenACC/OpenACCOpsAttributes.cpp.inc"
5030
5031#define GET_TYPEDEF_CLASSES
5032#include "mlir/Dialect/OpenACC/OpenACCOpsTypes.cpp.inc"
5033
5034//===----------------------------------------------------------------------===//
5035// acc dialect utilities
5036//===----------------------------------------------------------------------===//
5037
5038mlir::TypedValue<mlir::acc::PointerLikeType>
5039mlir::acc::getVarPtr(mlir::Operation *accDataClauseOp) {
5040 auto varPtr{llvm::TypeSwitch<mlir::Operation *,
5041 mlir::TypedValue<mlir::acc::PointerLikeType>>(
5042 accDataClauseOp)
5043 .Case<ACC_DATA_ENTRY_OPS, mlir::acc::MapInfoOp>(
5044 caseFn: [&](auto entry) { return entry.getVarPtr(); })
5045 .Case<mlir::acc::CopyoutOp, mlir::acc::UpdateHostOp>(
5046 caseFn: [&](auto exit) { return exit.getVarPtr(); })
5047 .Default(defaultFn: [&](mlir::Operation *) {
5048 return mlir::TypedValue<mlir::acc::PointerLikeType>();
5049 })};
5050 return varPtr;
5051}
5052
5053mlir::Value mlir::acc::getVar(mlir::Operation *accDataClauseOp) {
5054 auto varPtr{llvm::TypeSwitch<mlir::Operation *, mlir::Value>(accDataClauseOp)
5055 .Case<ACC_DATA_ENTRY_OPS, mlir::acc::MapInfoOp>(
5056 caseFn: [&](auto entry) { return entry.getVar(); })
5057 .Default(defaultFn: [&](mlir::Operation *) { return mlir::Value(); })};
5058 return varPtr;
5059}
5060
5061mlir::Type mlir::acc::getVarType(mlir::Operation *accDataClauseOp) {
5062 auto varType{llvm::TypeSwitch<mlir::Operation *, mlir::Type>(accDataClauseOp)
5063 .Case<ACC_DATA_ENTRY_OPS, mlir::acc::MapInfoOp>(
5064 caseFn: [&](auto entry) { return entry.getVarType(); })
5065 .Case<mlir::acc::CopyoutOp, mlir::acc::UpdateHostOp>(
5066 caseFn: [&](auto exit) { return exit.getVarType(); })
5067 .Default(defaultFn: [&](mlir::Operation *) { return mlir::Type(); })};
5068 return varType;
5069}
5070
5071mlir::TypedValue<mlir::acc::PointerLikeType>
5072mlir::acc::getAccPtr(mlir::Operation *accDataClauseOp) {
5073 auto accPtr{
5074 llvm::TypeSwitch<mlir::Operation *,
5075 mlir::TypedValue<mlir::acc::PointerLikeType>>(
5076 accDataClauseOp)
5077 .Case<ACC_DATA_ENTRY_OPS, ACC_DATA_EXIT_OPS, mlir::acc::MapInfoOp>(
5078 caseFn: [&](auto dataClause) { return dataClause.getAccPtr(); })
5079 .Default(defaultFn: [&](mlir::Operation *) {
5080 return mlir::TypedValue<mlir::acc::PointerLikeType>();
5081 })};
5082 return accPtr;
5083}
5084
5085mlir::Value mlir::acc::getAccVar(mlir::Operation *accDataClauseOp) {
5086 auto accPtr{
5087 llvm::TypeSwitch<mlir::Operation *, mlir::Value>(accDataClauseOp)
5088 .Case<ACC_DATA_ENTRY_OPS, ACC_DATA_EXIT_OPS, mlir::acc::MapInfoOp>(
5089 caseFn: [&](auto dataClause) { return dataClause.getAccVar(); })
5090 .Default(defaultFn: [&](mlir::Operation *) { return mlir::Value(); })};
5091 return accPtr;
5092}
5093
5094mlir::Value mlir::acc::getVarPtrPtr(mlir::Operation *accDataClauseOp) {
5095 auto varPtrPtr{
5096 llvm::TypeSwitch<mlir::Operation *, mlir::Value>(accDataClauseOp)
5097 .Case<ACC_DATA_ENTRY_OPS, mlir::acc::MapInfoOp>(
5098 caseFn: [&](auto dataClause) { return dataClause.getVarPtrPtr(); })
5099 .Default(defaultFn: [&](mlir::Operation *) { return mlir::Value(); })};
5100 return varPtrPtr;
5101}
5102
5103mlir::SmallVector<mlir::Value>
5104mlir::acc::getBounds(mlir::Operation *accDataClauseOp) {
5105 mlir::SmallVector<mlir::Value> bounds{
5106 llvm::TypeSwitch<mlir::Operation *, mlir::SmallVector<mlir::Value>>(
5107 accDataClauseOp)
5108 .Case<ACC_DATA_ENTRY_OPS, ACC_DATA_EXIT_OPS, mlir::acc::MapInfoOp>(
5109 caseFn: [&](auto dataClause) {
5110 return mlir::SmallVector<mlir::Value>(
5111 dataClause.getBounds().begin(),
5112 dataClause.getBounds().end());
5113 })
5114 .Default(defaultFn: [&](mlir::Operation *) {
5115 return mlir::SmallVector<mlir::Value, 0>();
5116 })};
5117 return bounds;
5118}
5119
5120mlir::SmallVector<mlir::Value>
5121mlir::acc::getAsyncOperands(mlir::Operation *accDataClauseOp) {
5122 return llvm::TypeSwitch<mlir::Operation *, mlir::SmallVector<mlir::Value>>(
5123 accDataClauseOp)
5124 .Case<ACC_DATA_ENTRY_OPS, ACC_DATA_EXIT_OPS>(caseFn: [&](auto dataClause) {
5125 return mlir::SmallVector<mlir::Value>(
5126 dataClause.getAsyncOperands().begin(),
5127 dataClause.getAsyncOperands().end());
5128 })
5129 .Default(defaultFn: [&](mlir::Operation *) {
5130 return mlir::SmallVector<mlir::Value, 0>();
5131 });
5132}
5133
5134mlir::ArrayAttr
5135mlir::acc::getAsyncOperandsDeviceType(mlir::Operation *accDataClauseOp) {
5136 return llvm::TypeSwitch<mlir::Operation *, mlir::ArrayAttr>(accDataClauseOp)
5137 .Case<ACC_DATA_ENTRY_OPS, ACC_DATA_EXIT_OPS>(caseFn: [&](auto dataClause) {
5138 return dataClause.getAsyncOperandsDeviceTypeAttr();
5139 })
5140 .Default(defaultFn: [&](mlir::Operation *) { return mlir::ArrayAttr{}; });
5141}
5142
5143mlir::ArrayAttr mlir::acc::getAsyncOnly(mlir::Operation *accDataClauseOp) {
5144 return llvm::TypeSwitch<mlir::Operation *, mlir::ArrayAttr>(accDataClauseOp)
5145 .Case<ACC_DATA_ENTRY_OPS, ACC_DATA_EXIT_OPS>(
5146 caseFn: [&](auto dataClause) { return dataClause.getAsyncOnlyAttr(); })
5147 .Default(defaultFn: [&](mlir::Operation *) { return mlir::ArrayAttr{}; });
5148}
5149
5150SmallVector<DeviceType, 3>
5151acc::getDeviceTypesByPrecedence(DeviceType deviceType) {
5152 SmallVector<DeviceType, 3> deviceTypes;
5153 if (deviceType != DeviceType::None && deviceType != DeviceType::Star)
5154 deviceTypes.push_back(Elt: deviceType);
5155 deviceTypes.push_back(Elt: DeviceType::Star);
5156 deviceTypes.push_back(Elt: DeviceType::None);
5157 return deviceTypes;
5158}
5159
5160std::optional<llvm::StringRef> mlir::acc::getVarName(mlir::Operation *accOp) {
5161 auto name{
5162 llvm::TypeSwitch<mlir::Operation *, std::optional<llvm::StringRef>>(accOp)
5163 .Case<ACC_DATA_ENTRY_OPS, mlir::acc::MapInfoOp>(
5164 caseFn: [&](auto entry) { return entry.getName(); })
5165 .Default(defaultFn: [&](mlir::Operation *) -> std::optional<llvm::StringRef> {
5166 return {};
5167 })};
5168 return name;
5169}
5170
5171std::optional<mlir::acc::DataClause>
5172mlir::acc::getDataClause(mlir::Operation *accDataEntryOp) {
5173 auto dataClause{
5174 llvm::TypeSwitch<mlir::Operation *, std::optional<mlir::acc::DataClause>>(
5175 accDataEntryOp)
5176 .Case<ACC_DATA_ENTRY_OPS>(
5177 caseFn: [&](auto entry) { return entry.getDataClause(); })
5178 .Default(defaultFn: [&](mlir::Operation *) { return std::nullopt; })};
5179 return dataClause;
5180}
5181
5182bool mlir::acc::isDataEntryOp(mlir::Operation *op) {
5183 return mlir::isa_and_nonnull<ACC_DATA_ENTRY_OPS, mlir::acc::MapInfoOp>(Val: op);
5184}
5185
5186bool mlir::acc::getImplicitFlag(mlir::Operation *accDataEntryOp) {
5187 return llvm::TypeSwitch<mlir::Operation *, bool>(accDataEntryOp)
5188 .Case<ACC_DATA_ENTRY_OPS>(caseFn: [&](auto entry) { return entry.getImplicit(); })
5189 .Case<mlir::acc::MapInfoOp>(caseFn: [&](auto mapInfo) {
5190 return bitEnumContainsAny(mapInfo.getMapFlags(),
5191 mlir::acc::MapFlags::implicit);
5192 })
5193 .Default(defaultFn: [&](mlir::Operation *) { return false; });
5194}
5195
5196bool mlir::acc::getSyntheticFlag(mlir::Operation *accDataClauseOp) {
5197 return llvm::TypeSwitch<mlir::Operation *, bool>(accDataClauseOp)
5198 .Case<ACC_DATA_CLAUSE_OPS>(
5199 caseFn: [&](auto dataClause) { return dataClause.getSynthetic(); })
5200 .Default(defaultFn: [&](mlir::Operation *) { return false; });
5201}
5202
5203mlir::ValueRange mlir::acc::getDataOperands(mlir::Operation *accOp) {
5204 auto dataOperands{
5205 llvm::TypeSwitch<mlir::Operation *, mlir::ValueRange>(accOp)
5206 .Case<ACC_COMPUTE_AND_DATA_CONSTRUCT_OPS,
5207 mlir::acc::KernelEnvironmentOp>(
5208 caseFn: [&](auto entry) { return entry.getDataClauseOperands(); })
5209 .Default(defaultFn: [&](mlir::Operation *) { return mlir::ValueRange(); })};
5210 return dataOperands;
5211}
5212
5213mlir::MutableOperandRange
5214mlir::acc::getMutableDataOperands(mlir::Operation *accOp) {
5215 auto dataOperands{
5216 llvm::TypeSwitch<mlir::Operation *, mlir::MutableOperandRange>(accOp)
5217 .Case<ACC_COMPUTE_AND_DATA_CONSTRUCT_OPS,
5218 mlir::acc::KernelEnvironmentOp>(
5219 caseFn: [&](auto entry) { return entry.getDataClauseOperandsMutable(); })
5220 .Default(defaultFn: [&](mlir::Operation *) { return nullptr; })};
5221 return dataOperands;
5222}
5223
5224mlir::SymbolRefAttr mlir::acc::getRecipe(mlir::Operation *accOp) {
5225 auto recipe{
5226 llvm::TypeSwitch<mlir::Operation *, mlir::SymbolRefAttr>(accOp)
5227 .Case<ACC_DATA_ENTRY_OPS>(
5228 caseFn: [&](auto entry) { return entry.getRecipeAttr(); })
5229 .Default(defaultFn: [&](mlir::Operation *) { return mlir::SymbolRefAttr{}; })};
5230 return recipe;
5231}
5232