1//===- OpenMPDialect.cpp - MLIR Dialect for OpenMP implementation ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the OpenMP dialect and its operations.
10//
11//===----------------------------------------------------------------------===//
12
13#include "mlir/Dialect/OpenMP/OpenMPDialect.h"
14#include "mlir/Conversion/ConvertToLLVM/ToLLVMInterface.h"
15#include "mlir/Dialect/Func/IR/FuncOps.h"
16#include "mlir/Dialect/LLVMIR/LLVMModuleOps.h"
17#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
18#include "mlir/Dialect/OpenMP/OpenMPClauseOperands.h"
19#include "mlir/IR/Attributes.h"
20#include "mlir/IR/BuiltinAttributes.h"
21#include "mlir/IR/Matchers.h"
22#include "mlir/IR/OpImplementation.h"
23#include "mlir/IR/OperationSupport.h"
24#include "mlir/IR/SymbolTable.h"
25#include "mlir/Interfaces/FoldInterfaces.h"
26#include "mlir/Interfaces/SideEffectInterfaces.h"
27
28#include "llvm/ADT/ArrayRef.h"
29#include "llvm/ADT/PostOrderIterator.h"
30#include "llvm/ADT/STLExtras.h"
31#include "llvm/ADT/STLForwardCompat.h"
32#include "llvm/ADT/SmallString.h"
33#include "llvm/ADT/StringRef.h"
34#include "llvm/ADT/TypeSwitch.h"
35#include "llvm/ADT/bit.h"
36#include "llvm/Support/InterleavedRange.h"
37#include <cstddef>
38#include <iterator>
39#include <optional>
40#include <variant>
41
42#include "mlir/Dialect/OpenMP/OpenMPOpsDialect.cpp.inc"
43#include "mlir/Dialect/OpenMP/OpenMPOpsEnums.cpp.inc"
44#include "mlir/Dialect/OpenMP/OpenMPOpsInterfaces.cpp.inc"
45#include "mlir/Dialect/OpenMP/OpenMPTypeInterfaces.cpp.inc"
46
47using namespace mlir;
48using namespace mlir::omp;
49
50static ArrayAttr makeArrayAttr(MLIRContext *context,
51 llvm::ArrayRef<Attribute> attrs) {
52 return attrs.empty() ? nullptr : ArrayAttr::get(context, value: attrs);
53}
54
55static DenseBoolArrayAttr
56makeDenseBoolArrayAttr(MLIRContext *ctx, const ArrayRef<bool> boolArray) {
57 return boolArray.empty() ? nullptr : DenseBoolArrayAttr::get(context: ctx, content: boolArray);
58}
59
60static DenseI64ArrayAttr
61makeDenseI64ArrayAttr(MLIRContext *ctx, const ArrayRef<int64_t> intArray) {
62 return intArray.empty() ? nullptr : DenseI64ArrayAttr::get(context: ctx, content: intArray);
63}
64
65namespace {
66struct MemRefPointerLikeModel
67 : public PointerLikeType::ExternalModel<MemRefPointerLikeModel,
68 MemRefType> {
69 Type getElementType(Type pointer) const {
70 return llvm::cast<MemRefType>(Val&: pointer).getElementType();
71 }
72};
73
74struct LLVMPointerPointerLikeModel
75 : public PointerLikeType::ExternalModel<LLVMPointerPointerLikeModel,
76 LLVM::LLVMPointerType> {
77 Type getElementType(Type pointer) const { return Type(); }
78};
79} // namespace
80
81/// Generate a name of a canonical loop nest of the format
82/// `<prefix>(_r<idx>_s<idx>)*`. Hereby, `_r<idx>` identifies the region
83/// argument index of an operation that has multiple regions, if the operation
84/// has multiple regions.
85/// `_s<idx>` identifies the position of an operation within a region, where
86/// only operations that may potentially contain loops ("container operations"
87/// i.e. have region arguments) are counted. Again, it is omitted if there is
88/// only one such operation in a region. If there are canonical loops nested
89/// inside each other, also may also use the format `_d<num>` where <num> is the
90/// nesting depth of the loop.
91///
92/// The generated name is a best-effort to make canonical loop unique within an
93/// SSA namespace. This also means that regions with IsolatedFromAbove property
94/// do not consider any parents or siblings.
95static std::string generateLoopNestingName(StringRef prefix,
96 CanonicalLoopOp op) {
97 struct Component {
98 /// If true, this component describes a region operand of an operation (the
99 /// operand's owner) If false, this component describes an operation located
100 /// in a parent region
101 bool isRegionArgOfOp;
102 bool skip = false;
103 bool isUnique = false;
104
105 size_t idx;
106 Operation *op;
107 Region *parentRegion;
108 size_t loopDepth;
109
110 Operation *&getOwnerOp() {
111 assert(isRegionArgOfOp && "Must describe a region operand");
112 return op;
113 }
114 size_t &getArgIdx() {
115 assert(isRegionArgOfOp && "Must describe a region operand");
116 return idx;
117 }
118
119 Operation *&getContainerOp() {
120 assert(!isRegionArgOfOp && "Must describe a operation of a region");
121 return op;
122 }
123 size_t &getOpPos() {
124 assert(!isRegionArgOfOp && "Must describe a operation of a region");
125 return idx;
126 }
127 bool isLoopOp() const {
128 assert(!isRegionArgOfOp && "Must describe a operation of a region");
129 return isa<CanonicalLoopOp>(Val: op);
130 }
131 Region *&getParentRegion() {
132 assert(!isRegionArgOfOp && "Must describe a operation of a region");
133 return parentRegion;
134 }
135 size_t &getLoopDepth() {
136 assert(!isRegionArgOfOp && "Must describe a operation of a region");
137 return loopDepth;
138 }
139
140 void skipIf(bool v = true) { skip = skip || v; }
141 };
142
143 // List of ancestors, from inner to outer.
144 // Alternates between
145 // * region argument of an operation
146 // * operation within a region
147 SmallVector<Component> components;
148
149 // Gather a list of parent regions and operations, and the position within
150 // their parent
151 Operation *o = op.getOperation();
152 while (o) {
153 // Operation within a region
154 Region *r = o->getParentRegion();
155 if (!r)
156 break;
157
158 llvm::ReversePostOrderTraversal<Block *> traversal(&r->getBlocks().front());
159 size_t idx = 0;
160 bool found = false;
161 size_t sequentialIdx = -1;
162 bool isOnlyContainerOp = true;
163 for (Block *b : traversal) {
164 for (Operation &op : *b) {
165 if (&op == o && !found) {
166 sequentialIdx = idx;
167 found = true;
168 }
169 if (op.getNumRegions()) {
170 idx += 1;
171 if (idx > 1)
172 isOnlyContainerOp = false;
173 }
174 if (found && !isOnlyContainerOp)
175 break;
176 }
177 }
178
179 Component &containerOpInRegion = components.emplace_back();
180 containerOpInRegion.isRegionArgOfOp = false;
181 containerOpInRegion.isUnique = isOnlyContainerOp;
182 containerOpInRegion.getContainerOp() = o;
183 containerOpInRegion.getOpPos() = sequentialIdx;
184 containerOpInRegion.getParentRegion() = r;
185
186 Operation *parent = r->getParentOp();
187
188 // Region argument of an operation
189 Component &regionArgOfOperation = components.emplace_back();
190 regionArgOfOperation.isRegionArgOfOp = true;
191 regionArgOfOperation.isUnique = true;
192 regionArgOfOperation.getArgIdx() = 0;
193 regionArgOfOperation.getOwnerOp() = parent;
194
195 // The IsolatedFromAbove trait of the parent operation implies that each
196 // individual region argument has its own separate namespace, so no
197 // ambiguity.
198 if (!parent || parent->hasTrait<mlir::OpTrait::IsIsolatedFromAbove>())
199 break;
200
201 // Component only needed if operation has multiple region operands. Region
202 // arguments may be optional, but we currently do not consider this.
203 if (parent->getRegions().size() > 1) {
204 auto getRegionIndex = [](Operation *o, Region *r) {
205 for (auto [idx, region] : llvm::enumerate(First: o->getRegions())) {
206 if (&region == r)
207 return idx;
208 }
209 llvm_unreachable("Region not child of its parent operation");
210 };
211 regionArgOfOperation.isUnique = false;
212 regionArgOfOperation.getArgIdx() = getRegionIndex(parent, r);
213 }
214
215 // next parent
216 o = parent;
217 }
218
219 // Determine whether a region-argument component is not needed
220 for (Component &c : components)
221 c.skipIf(v: c.isRegionArgOfOp && c.isUnique);
222
223 // Find runs of nested loops and determine each loop's depth in the loop nest
224 size_t numSurroundingLoops = 0;
225 for (Component &c : llvm::reverse(C&: components)) {
226 if (c.skip)
227 continue;
228
229 // non-skipped multi-argument operands interrupt the loop nest
230 if (c.isRegionArgOfOp) {
231 numSurroundingLoops = 0;
232 continue;
233 }
234
235 // Multiple loops in a region means each of them is the outermost loop of a
236 // new loop nest
237 if (!c.isUnique)
238 numSurroundingLoops = 0;
239
240 c.getLoopDepth() = numSurroundingLoops;
241
242 // Next loop is surrounded by one more loop
243 if (isa<CanonicalLoopOp>(Val: c.getContainerOp()))
244 numSurroundingLoops += 1;
245 }
246
247 // In loop nests, skip all but the innermost loop that contains the depth
248 // number
249 bool isLoopNest = false;
250 for (Component &c : components) {
251 if (c.skip || c.isRegionArgOfOp)
252 continue;
253
254 if (!isLoopNest && c.getLoopDepth() >= 1) {
255 // Innermost loop of a loop nest of at least two loops
256 isLoopNest = true;
257 } else if (isLoopNest) {
258 // Non-innermost loop of a loop nest
259 c.skipIf(v: c.isUnique);
260
261 // If there is no surrounding loop left, this must have been the outermost
262 // loop; leave loop-nest mode for the next iteration
263 if (c.getLoopDepth() == 0)
264 isLoopNest = false;
265 }
266 }
267
268 // Skip non-loop unambiguous regions (but they should interrupt loop nests, so
269 // we mark them as skipped only after computing loop nests)
270 for (Component &c : components)
271 c.skipIf(v: !c.isRegionArgOfOp && c.isUnique &&
272 !isa<CanonicalLoopOp>(Val: c.getContainerOp()));
273
274 // Components can be skipped if they are already disambiguated by their parent
275 // (or does not have a parent)
276 bool newRegion = true;
277 for (Component &c : llvm::reverse(C&: components)) {
278 c.skipIf(v: newRegion && c.isUnique);
279
280 // non-skipped components disambiguate unique children
281 if (!c.skip)
282 newRegion = true;
283
284 // ...except canonical loops that need a suffix for each nest
285 if (!c.isRegionArgOfOp && c.getContainerOp())
286 newRegion = false;
287 }
288
289 // Compile the nesting name string
290 SmallString<64> Name{prefix};
291 llvm::raw_svector_ostream NameOS(Name);
292 for (auto &c : llvm::reverse(C&: components)) {
293 if (c.skip)
294 continue;
295
296 if (c.isRegionArgOfOp)
297 NameOS << "_r" << c.getArgIdx();
298 else if (c.getLoopDepth() >= 1)
299 NameOS << "_d" << c.getLoopDepth();
300 else
301 NameOS << "_s" << c.getOpPos();
302 }
303
304 return NameOS.str().str();
305}
306
307void OpenMPDialect::initialize() {
308 addOperations<
309#define GET_OP_LIST
310#include "mlir/Dialect/OpenMP/OpenMPOps.cpp.inc"
311 >();
312 addAttributes<
313#define GET_ATTRDEF_LIST
314#include "mlir/Dialect/OpenMP/OpenMPOpsAttributes.cpp.inc"
315 >();
316 addTypes<
317#define GET_TYPEDEF_LIST
318#include "mlir/Dialect/OpenMP/OpenMPOpsTypes.cpp.inc"
319 >();
320
321 declarePromisedInterface<ConvertToLLVMPatternInterface, OpenMPDialect>();
322
323 MemRefType::attachInterface<MemRefPointerLikeModel>(context&: *getContext());
324 LLVM::LLVMPointerType::attachInterface<LLVMPointerPointerLikeModel>(
325 context&: *getContext());
326
327 // Attach default offload module interface to module op to access
328 // offload functionality through
329 mlir::ModuleOp::attachInterface<mlir::omp::OffloadModuleDefaultModel>(
330 context&: *getContext());
331
332 // Attach default declare target interfaces to operations which can be marked
333 // as declare target (Global Operations and Functions/Subroutines in dialects
334 // that Fortran (or other languages that lower to MLIR) translates too
335 mlir::LLVM::GlobalOp::attachInterface<
336 mlir::omp::DeclareTargetDefaultModel<mlir::LLVM::GlobalOp>>(
337 context&: *getContext());
338 mlir::LLVM::LLVMFuncOp::attachInterface<
339 mlir::omp::DeclareTargetDefaultModel<mlir::LLVM::LLVMFuncOp>>(
340 context&: *getContext());
341 mlir::func::FuncOp::attachInterface<
342 mlir::omp::DeclareTargetDefaultModel<mlir::func::FuncOp>>(context&: *getContext());
343}
344
345//===----------------------------------------------------------------------===//
346// Dialect operation attribute verification
347//===----------------------------------------------------------------------===//
348
349static LogicalResult verifyDeclareTargetAttr(Operation *op, Attribute attr) {
350 if (!isa<DeclareTargetInterface>(Val: op))
351 return op->emitError() << "omp.declare_target can only be applied to "
352 "DeclareTargetInterface ops";
353
354 auto declareTargetAttr = dyn_cast<DeclareTargetAttr>(Val&: attr);
355 if (!declareTargetAttr)
356 return op->emitError()
357 << "omp.declare_target must be an #omp.declaretarget attribute";
358
359 if (isa<mlir::FunctionOpInterface>(Val: op)) {
360 if (declareTargetAttr.getAutomap())
361 return op->emitOpError()
362 << "omp.declare_target 'automap' is not valid on functions";
363
364 // TODO: Disallow the `local` clause (OpenMP 6.0).
365 if (declareTargetAttr.getCaptureClause() ==
366 mlir::omp::DeclareTargetCaptureClause::link)
367 return op->emitOpError()
368 << "omp.declare_target 'link' is not valid on functions";
369 } else {
370 // TODO: Disallow the `indirect` clause (OpenMP 5.1).
371 if (declareTargetAttr.getImplicit())
372 return op->emitOpError()
373 << "omp.declare_target 'implicit' is only valid on functions";
374 }
375 return success();
376}
377
378LogicalResult
379OpenMPDialect::verifyOperationAttribute(Operation *op,
380 NamedAttribute attribute) {
381 if (attribute.getName() == "omp.declare_target")
382 return verifyDeclareTargetAttr(op, attr: attribute.getValue());
383
384 return success();
385}
386
387//===----------------------------------------------------------------------===//
388// Parser and printer for Allocate Clause
389//===----------------------------------------------------------------------===//
390
391/// Parse an allocate clause with allocators and a list of operands with types.
392///
393/// allocate-operand-list :: = allocate-operand |
394/// allocator-operand `,` allocate-operand-list
395/// allocate-operand :: = ssa-id-and-type -> ssa-id-and-type
396/// ssa-id-and-type ::= ssa-id `:` type
397static ParseResult parseAllocateAndAllocator(
398 OpAsmParser &parser,
399 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &allocateVars,
400 SmallVectorImpl<Type> &allocateTypes,
401 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &allocatorVars,
402 SmallVectorImpl<Type> &allocatorTypes) {
403
404 return parser.parseCommaSeparatedList(parseElementFn: [&]() {
405 OpAsmParser::UnresolvedOperand operand;
406 Type type;
407 if (parser.parseOperand(result&: operand) || parser.parseColonType(result&: type))
408 return failure();
409 allocatorVars.push_back(Elt: operand);
410 allocatorTypes.push_back(Elt: type);
411 if (parser.parseArrow())
412 return failure();
413 if (parser.parseOperand(result&: operand) || parser.parseColonType(result&: type))
414 return failure();
415
416 allocateVars.push_back(Elt: operand);
417 allocateTypes.push_back(Elt: type);
418 return success();
419 });
420}
421
422/// Print allocate clause
423static void printAllocateAndAllocator(OpAsmPrinter &p, Operation *op,
424 OperandRange allocateVars,
425 TypeRange allocateTypes,
426 OperandRange allocatorVars,
427 TypeRange allocatorTypes) {
428 for (unsigned i = 0; i < allocateVars.size(); ++i) {
429 std::string separator = i == allocateVars.size() - 1 ? "" : ", ";
430 p << allocatorVars[i] << " : " << allocatorTypes[i] << " -> ";
431 p << allocateVars[i] << " : " << allocateTypes[i] << separator;
432 }
433}
434
435//===----------------------------------------------------------------------===//
436// Parser and printer for a clause attribute (StringEnumAttr)
437//===----------------------------------------------------------------------===//
438
439template <typename ClauseAttr>
440static ParseResult parseClauseAttr(AsmParser &parser, ClauseAttr &attr) {
441 using ClauseT = decltype(std::declval<ClauseAttr>().getValue());
442 StringRef enumStr;
443 SMLoc loc = parser.getCurrentLocation();
444 if (parser.parseKeyword(keyword: &enumStr))
445 return failure();
446 if (std::optional<ClauseT> enumValue = symbolizeEnum<ClauseT>(enumStr)) {
447 attr = ClauseAttr::get(parser.getContext(), *enumValue);
448 return success();
449 }
450 return parser.emitError(loc, message: "invalid clause value: '") << enumStr << "'";
451}
452
453template <typename ClauseAttr>
454static void printClauseAttr(OpAsmPrinter &p, Operation *op, ClauseAttr attr) {
455 p << stringifyEnum(attr.getValue());
456}
457
458//===----------------------------------------------------------------------===//
459// Parser and printer for Linear Clause
460//===----------------------------------------------------------------------===//
461
462/// linear ::= `linear` `(` linear-list `)`
463/// linear-list := linear-val | linear-val linear-list
464/// linear-val := ssa-id-and-type `=` ssa-id-and-type
465/// | `val` `(` ssa-id-and-type `=` ssa-id-and-type `)`
466/// | `ref` `(` ssa-id-and-type `=` ssa-id-and-type `)`
467/// | `uval` `(` ssa-id-and-type `=` ssa-id-and-type `)`
468static ParseResult parseLinearClause(
469 OpAsmParser &parser,
470 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &linearVars,
471 SmallVectorImpl<Type> &linearTypes,
472 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &linearStepVars,
473 SmallVectorImpl<Type> &linearStepTypes, ArrayAttr &linearModifiers,
474 ArrayAttr &linearSyms) {
475 SmallVector<Attribute> modifiers;
476 auto result = parser.parseCommaSeparatedList(parseElementFn: [&]() {
477 OpAsmParser::UnresolvedOperand var;
478 Type type, stepType;
479 OpAsmParser::UnresolvedOperand stepVar;
480
481 std::optional<omp::LinearModifier> linearModifier;
482 if (succeeded(Result: parser.parseOptionalKeyword(keyword: "val"))) {
483 linearModifier = omp::LinearModifier::val;
484 } else if (succeeded(Result: parser.parseOptionalKeyword(keyword: "ref"))) {
485 linearModifier = omp::LinearModifier::ref;
486 } else if (succeeded(Result: parser.parseOptionalKeyword(keyword: "uval"))) {
487 linearModifier = omp::LinearModifier::uval;
488 }
489
490 bool hasLinearModifierParens = linearModifier.has_value();
491 if (hasLinearModifierParens && parser.parseLParen())
492 return failure();
493
494 if (parser.parseOperand(result&: var) || parser.parseColonType(result&: type) ||
495 parser.parseEqual() || parser.parseOperand(result&: stepVar) ||
496 parser.parseColonType(result&: stepType))
497 return failure();
498
499 if (hasLinearModifierParens && parser.parseRParen())
500 return failure();
501
502 linearVars.push_back(Elt: var);
503 linearTypes.push_back(Elt: type);
504 linearStepVars.push_back(Elt: stepVar);
505 linearStepTypes.push_back(Elt: stepType);
506 if (linearModifier) {
507 modifiers.push_back(
508 Elt: omp::LinearModifierAttr::get(context: parser.getContext(), value: *linearModifier));
509 } else {
510 modifiers.push_back(Elt: UnitAttr::get(context: parser.getContext()));
511 }
512 return success();
513 });
514 if (failed(Result: result))
515 return failure();
516 linearModifiers = ArrayAttr::get(context: parser.getContext(), value: modifiers);
517 return success();
518}
519
520/// Print Linear Clause. `linearSyms` is only used by ops with a region, which
521/// print the linear clause with their entry block arguments.
522static void printLinearClause(OpAsmPrinter &p, Operation *op,
523 ValueRange linearVars, TypeRange linearTypes,
524 ValueRange linearStepVars, TypeRange stepVarTypes,
525 ArrayAttr linearModifiers, ArrayAttr linearSyms) {
526 size_t linearVarsSize = linearVars.size();
527 for (unsigned i = 0; i < linearVarsSize; ++i) {
528 if (i != 0)
529 p << ", ";
530 // Print modifier keyword wrapper if present.
531 Attribute modAttr = linearModifiers ? linearModifiers[i] : nullptr;
532 auto mod = modAttr ? dyn_cast<omp::LinearModifierAttr>(Val&: modAttr) : nullptr;
533 if (mod) {
534 p << omp::stringifyLinearModifier(val: mod.getValue()) << "(";
535 }
536 p << linearVars[i] << " : " << linearTypes[i];
537 p << " = " << linearStepVars[i] << " : " << stepVarTypes[i];
538 if (mod)
539 p << ")";
540 }
541}
542
543//===----------------------------------------------------------------------===//
544// Verifier for Linear modifier
545//===----------------------------------------------------------------------===//
546
547/// OpenMP 5.2, Section 5.4.6: "A linear-modifier may be specified as ref or
548/// uval only on a declare simd directive."
549/// Also verifies that modifier count matches variable count.
550static LogicalResult
551verifyLinearModifiers(Operation *op, std::optional<ArrayAttr> linearModifiers,
552 OperandRange linearVars, bool isDeclareSimd = false) {
553 if (!linearModifiers)
554 return success();
555 if (linearModifiers->size() != linearVars.size())
556 return op->emitOpError()
557 << "expected as many linear modifiers as linear variables";
558 if (!isDeclareSimd) {
559 for (Attribute attr : *linearModifiers) {
560 if (!attr)
561 continue;
562 auto modAttr = dyn_cast<omp::LinearModifierAttr>(Val&: attr);
563 if (!modAttr)
564 continue;
565 omp::LinearModifier mod = modAttr.getValue();
566 if (mod == omp::LinearModifier::ref || mod == omp::LinearModifier::uval)
567 return op->emitOpError()
568 << "linear modifier '" << omp::stringifyLinearModifier(val: mod)
569 << "' may only be specified on a declare simd directive";
570 }
571 }
572 return success();
573}
574
575//===----------------------------------------------------------------------===//
576// Verifier for Nontemporal Clause
577//===----------------------------------------------------------------------===//
578
579static LogicalResult verifyNontemporalClause(Operation *op,
580 OperandRange nontemporalVars) {
581
582 // Check if each var is unique - OpenMP 5.0 -> 2.9.3.1 section
583 DenseSet<Value> nontemporalItems;
584 for (const auto &it : nontemporalVars)
585 if (!nontemporalItems.insert(V: it).second)
586 return op->emitOpError() << "nontemporal variable used more than once";
587
588 return success();
589}
590
591//===----------------------------------------------------------------------===//
592// Parser, verifier and printer for Aligned Clause
593//===----------------------------------------------------------------------===//
594static LogicalResult verifyAlignedClause(Operation *op,
595 std::optional<ArrayAttr> alignments,
596 OperandRange alignedVars) {
597 // Check if number of alignment values equals to number of aligned variables
598 if (!alignedVars.empty()) {
599 if (!alignments || alignments->size() != alignedVars.size())
600 return op->emitOpError()
601 << "expected as many alignment values as aligned variables";
602 } else {
603 if (alignments)
604 return op->emitOpError() << "unexpected alignment values attribute";
605 return success();
606 }
607
608 // Check if each var is aligned only once - OpenMP 4.5 -> 2.8.1 section
609 DenseSet<Value> alignedItems;
610 for (auto it : alignedVars)
611 if (!alignedItems.insert(V: it).second)
612 return op->emitOpError() << "aligned variable used more than once";
613
614 if (!alignments)
615 return success();
616
617 // Check if all alignment values are positive - OpenMP 4.5 -> 2.8.1 section
618 for (unsigned i = 0; i < (*alignments).size(); ++i) {
619 if (auto intAttr = llvm::dyn_cast<IntegerAttr>(Val: (*alignments)[i])) {
620 if (intAttr.getValue().sle(RHS: 0))
621 return op->emitOpError() << "alignment should be greater than 0";
622 } else {
623 return op->emitOpError() << "expected integer alignment";
624 }
625 }
626
627 return success();
628}
629
630/// aligned ::= `aligned` `(` aligned-list `)`
631/// aligned-list := aligned-val | aligned-val aligned-list
632/// aligned-val := ssa-id-and-type `->` alignment
633static ParseResult
634parseAlignedClause(OpAsmParser &parser,
635 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &alignedVars,
636 SmallVectorImpl<Type> &alignedTypes,
637 ArrayAttr &alignmentsAttr) {
638 SmallVector<Attribute> alignmentVec;
639 if (failed(Result: parser.parseCommaSeparatedList(parseElementFn: [&]() {
640 if (parser.parseOperand(result&: alignedVars.emplace_back()) ||
641 parser.parseColonType(result&: alignedTypes.emplace_back()) ||
642 parser.parseArrow() ||
643 parser.parseAttribute(result&: alignmentVec.emplace_back())) {
644 return failure();
645 }
646 return success();
647 })))
648 return failure();
649 SmallVector<Attribute> alignments(alignmentVec.begin(), alignmentVec.end());
650 alignmentsAttr = ArrayAttr::get(context: parser.getContext(), value: alignments);
651 return success();
652}
653
654/// Print Aligned Clause
655static void printAlignedClause(OpAsmPrinter &p, Operation *op,
656 ValueRange alignedVars, TypeRange alignedTypes,
657 std::optional<ArrayAttr> alignments) {
658 for (unsigned i = 0; i < alignedVars.size(); ++i) {
659 if (i != 0)
660 p << ", ";
661 p << alignedVars[i] << " : " << alignedVars[i].getType();
662 p << " -> " << (*alignments)[i];
663 }
664}
665
666static LogicalResult verifyAllocateClause(
667 Operation *op, ValueRange allocateVars, ValueRange allocatorVars,
668 DenseI64ArrayAttr allocateAlignments,
669 DenseI64ArrayAttr allocatePrivateIndices, ValueRange privateVars = {},
670 ArrayAttr privateSyms = nullptr, bool requirePrivateIndices = false) {
671 if (allocateVars.size() != allocatorVars.size())
672 return op->emitError(
673 message: "expected equal sizes for allocate and allocator variables");
674
675 if (allocateVars.empty()) {
676 if (allocateAlignments)
677 return op->emitError(
678 message: "unexpected allocate alignments without allocate variables");
679 if (allocatePrivateIndices)
680 return op->emitError(
681 message: "unexpected allocate private indices without allocate variables");
682 return success();
683 }
684
685 if (allocateAlignments) {
686 ArrayRef<int64_t> alignments = allocateAlignments.asArrayRef();
687 if (alignments.size() != allocateVars.size())
688 return op->emitError(
689 message: "expected as many allocate alignments as allocate variables");
690 for (int64_t alignment : alignments) {
691 if (alignment < 0)
692 return op->emitError(message: "expected non-negative allocate alignments");
693 if (alignment != 0 && (alignment & (alignment - 1)) != 0)
694 return op->emitError(
695 message: "expected positive allocate alignments to be powers of two");
696 }
697 }
698
699 if (!allocatePrivateIndices) {
700 if (requirePrivateIndices)
701 return op->emitError(
702 message: "expected an allocate private index for each allocate variable");
703 return success();
704 }
705
706 ArrayRef<int64_t> indices = allocatePrivateIndices.asArrayRef();
707 if (indices.size() != allocateVars.size())
708 return op->emitError(
709 message: "expected as many allocate private indices as allocate variables");
710
711 DenseSet<int64_t> usedPrivateSlots;
712 for (auto [allocateVar, privateIndex] :
713 llvm::zip_equal(t&: allocateVars, u&: indices)) {
714 if (privateIndex < 0 ||
715 static_cast<uint64_t>(privateIndex) >= privateVars.size())
716 return op->emitError(message: "allocate private index is out of range");
717 if (!usedPrivateSlots.insert(V: privateIndex).second)
718 return op->emitError(
719 message: "allocate private index refers to a private variable more than once");
720
721 Value privateVar = privateVars[privateIndex];
722 if (allocateVar.getType() != privateVar.getType())
723 return op->emitError()
724 << "type mismatch between allocate variable and private variable "
725 "at index "
726 << privateIndex;
727 if (allocateVar != privateVar)
728 return op->emitError()
729 << "allocate variable does not match private variable at index "
730 << privateIndex;
731
732 if (!privateSyms ||
733 static_cast<uint64_t>(privateIndex) >= privateSyms.size())
734 return op->emitError(
735 message: "allocate private index does not have a privatizer symbol");
736
737 auto privateSym = dyn_cast<SymbolRefAttr>(Val: privateSyms[privateIndex]);
738 if (!privateSym)
739 return op->emitError(
740 message: "allocate private index does not reference a privatizer symbol");
741 PrivateClauseOp privatizer =
742 SymbolTable::lookupNearestSymbolFrom<PrivateClauseOp>(from: op, symbol: privateSym);
743 if (!privatizer)
744 return op->emitError() << "failed to lookup privatizer op with symbol: '"
745 << privateSym << "'";
746 if (privatizer.getDataSharingType() != DataSharingClauseType::Private &&
747 privatizer.getDataSharingType() != DataSharingClauseType::FirstPrivate)
748 return op->emitError(
749 message: "allocate private index must refer to private or firstprivate "
750 "storage");
751 }
752
753 return success();
754}
755
756//===----------------------------------------------------------------------===//
757// Parser, printer and verifier for Schedule Clause
758//===----------------------------------------------------------------------===//
759
760static ParseResult
761verifyScheduleModifiers(OpAsmParser &parser,
762 SmallVectorImpl<SmallString<12>> &modifiers) {
763 if (modifiers.size() > 2)
764 return parser.emitError(loc: parser.getNameLoc()) << " unexpected modifier(s)";
765 for (const auto &mod : modifiers) {
766 // Translate the string. If it has no value, then it was not a valid
767 // modifier!
768 auto symbol = symbolizeScheduleModifier(str: mod);
769 if (!symbol)
770 return parser.emitError(loc: parser.getNameLoc())
771 << " unknown modifier type: " << mod;
772 }
773
774 // If we have one modifier that is "simd", then stick a "none" modiifer in
775 // index 0.
776 if (modifiers.size() == 1) {
777 if (symbolizeScheduleModifier(str: modifiers[0]) == ScheduleModifier::simd) {
778 modifiers.push_back(Elt: modifiers[0]);
779 modifiers[0] = stringifyScheduleModifier(val: ScheduleModifier::none);
780 }
781 } else if (modifiers.size() == 2) {
782 // If there are two modifier:
783 // First modifier should not be simd, second one should be simd
784 if (symbolizeScheduleModifier(str: modifiers[0]) == ScheduleModifier::simd ||
785 symbolizeScheduleModifier(str: modifiers[1]) != ScheduleModifier::simd)
786 return parser.emitError(loc: parser.getNameLoc())
787 << " incorrect modifier order";
788 }
789 return success();
790}
791
792/// schedule ::= `schedule` `(` sched-list `)`
793/// sched-list ::= sched-val | sched-val sched-list |
794/// sched-val `,` sched-modifier
795/// sched-val ::= sched-with-chunk | sched-wo-chunk
796/// sched-with-chunk ::= sched-with-chunk-types (`=` ssa-id-and-type)?
797/// sched-with-chunk-types ::= `static` | `dynamic` | `guided`
798/// sched-wo-chunk ::= `auto` | `runtime`
799/// sched-modifier ::= sched-mod-val | sched-mod-val `,` sched-mod-val
800/// sched-mod-val ::= `monotonic` | `nonmonotonic` | `simd` | `none`
801static ParseResult
802parseScheduleClause(OpAsmParser &parser, ClauseScheduleKindAttr &scheduleAttr,
803 ScheduleModifierAttr &scheduleMod, UnitAttr &scheduleSimd,
804 std::optional<OpAsmParser::UnresolvedOperand> &chunkSize,
805 Type &chunkType) {
806 StringRef keyword;
807 if (parser.parseKeyword(keyword: &keyword))
808 return failure();
809 std::optional<mlir::omp::ClauseScheduleKind> schedule =
810 symbolizeClauseScheduleKind(str: keyword);
811 if (!schedule)
812 return parser.emitError(loc: parser.getNameLoc()) << " expected schedule kind";
813
814 scheduleAttr = ClauseScheduleKindAttr::get(context: parser.getContext(), value: *schedule);
815 switch (*schedule) {
816 case ClauseScheduleKind::Static:
817 case ClauseScheduleKind::Dynamic:
818 case ClauseScheduleKind::Guided:
819 if (succeeded(Result: parser.parseOptionalEqual())) {
820 chunkSize = OpAsmParser::UnresolvedOperand{};
821 if (parser.parseOperand(result&: *chunkSize) || parser.parseColonType(result&: chunkType))
822 return failure();
823 } else {
824 chunkSize = std::nullopt;
825 }
826 break;
827 case ClauseScheduleKind::Auto:
828 case ClauseScheduleKind::Runtime:
829 case ClauseScheduleKind::Distribute:
830 chunkSize = std::nullopt;
831 }
832
833 // If there is a comma, we have one or more modifiers..
834 SmallVector<SmallString<12>> modifiers;
835 while (succeeded(Result: parser.parseOptionalComma())) {
836 StringRef mod;
837 if (parser.parseKeyword(keyword: &mod))
838 return failure();
839 modifiers.push_back(Elt: mod);
840 }
841
842 if (verifyScheduleModifiers(parser, modifiers))
843 return failure();
844
845 if (!modifiers.empty()) {
846 SMLoc loc = parser.getCurrentLocation();
847 if (std::optional<ScheduleModifier> mod =
848 symbolizeScheduleModifier(str: modifiers[0])) {
849 scheduleMod = ScheduleModifierAttr::get(context: parser.getContext(), value: *mod);
850 } else {
851 return parser.emitError(loc, message: "invalid schedule modifier");
852 }
853 // Only SIMD attribute is allowed here!
854 if (modifiers.size() > 1) {
855 assert(symbolizeScheduleModifier(modifiers[1]) == ScheduleModifier::simd);
856 scheduleSimd = UnitAttr::get(context: parser.getBuilder().getContext());
857 }
858 }
859
860 return success();
861}
862
863/// Print schedule clause
864static void printScheduleClause(OpAsmPrinter &p, Operation *op,
865 ClauseScheduleKindAttr scheduleKind,
866 ScheduleModifierAttr scheduleMod,
867 UnitAttr scheduleSimd, Value scheduleChunk,
868 Type scheduleChunkType) {
869 p << stringifyClauseScheduleKind(val: scheduleKind.getValue());
870 if (scheduleChunk)
871 p << " = " << scheduleChunk << " : " << scheduleChunk.getType();
872 if (scheduleMod)
873 p << ", " << stringifyScheduleModifier(val: scheduleMod.getValue());
874 if (scheduleSimd)
875 p << ", simd";
876}
877
878//===----------------------------------------------------------------------===//
879// Parser and printer for Order Clause
880//===----------------------------------------------------------------------===//
881
882// order ::= `order` `(` [order-modifier ':'] concurrent `)`
883// order-modifier ::= reproducible | unconstrained
884static ParseResult parseOrderClause(OpAsmParser &parser,
885 ClauseOrderKindAttr &order,
886 OrderModifierAttr &orderMod) {
887 StringRef enumStr;
888 SMLoc loc = parser.getCurrentLocation();
889 if (parser.parseKeyword(keyword: &enumStr))
890 return failure();
891 if (std::optional<OrderModifier> enumValue =
892 symbolizeOrderModifier(str: enumStr)) {
893 orderMod = OrderModifierAttr::get(context: parser.getContext(), value: *enumValue);
894 if (parser.parseOptionalColon())
895 return failure();
896 loc = parser.getCurrentLocation();
897 if (parser.parseKeyword(keyword: &enumStr))
898 return failure();
899 }
900 if (std::optional<ClauseOrderKind> enumValue =
901 symbolizeClauseOrderKind(str: enumStr)) {
902 order = ClauseOrderKindAttr::get(context: parser.getContext(), value: *enumValue);
903 return success();
904 }
905 return parser.emitError(loc, message: "invalid clause value: '") << enumStr << "'";
906}
907
908static void printOrderClause(OpAsmPrinter &p, Operation *op,
909 ClauseOrderKindAttr order,
910 OrderModifierAttr orderMod) {
911 if (orderMod)
912 p << stringifyOrderModifier(val: orderMod.getValue()) << ":";
913 if (order)
914 p << stringifyClauseOrderKind(val: order.getValue());
915}
916
917template <typename ClauseTypeAttr, typename ClauseType>
918static ParseResult
919parseGranularityClause(OpAsmParser &parser, ClauseTypeAttr &prescriptiveness,
920 std::optional<OpAsmParser::UnresolvedOperand> &operand,
921 Type &operandType,
922 std::optional<ClauseType> (*symbolizeClause)(StringRef),
923 StringRef clauseName) {
924 StringRef enumStr;
925 if (succeeded(Result: parser.parseOptionalKeyword(keyword: &enumStr))) {
926 if (std::optional<ClauseType> enumValue = symbolizeClause(enumStr)) {
927 prescriptiveness = ClauseTypeAttr::get(parser.getContext(), *enumValue);
928 if (parser.parseComma())
929 return failure();
930 } else {
931 return parser.emitError(loc: parser.getCurrentLocation())
932 << "invalid " << clauseName << " modifier : '" << enumStr << "'";
933 ;
934 }
935 }
936
937 OpAsmParser::UnresolvedOperand var;
938 if (succeeded(Result: parser.parseOperand(result&: var))) {
939 operand = var;
940 } else {
941 return parser.emitError(loc: parser.getCurrentLocation())
942 << "expected " << clauseName << " operand";
943 }
944
945 if (operand.has_value()) {
946 if (parser.parseColonType(result&: operandType))
947 return failure();
948 }
949
950 return success();
951}
952
953template <typename ClauseTypeAttr, typename ClauseType>
954static void
955printGranularityClause(OpAsmPrinter &p, Operation *op,
956 ClauseTypeAttr prescriptiveness, Value operand,
957 mlir::Type operandType,
958 StringRef (*stringifyClauseType)(ClauseType)) {
959
960 if (prescriptiveness)
961 p << stringifyClauseType(prescriptiveness.getValue()) << ", ";
962
963 if (operand)
964 p << operand << ": " << operandType;
965}
966
967//===----------------------------------------------------------------------===//
968// Parser and printer for grainsize Clause
969//===----------------------------------------------------------------------===//
970
971// grainsize ::= `grainsize` `(` [strict ':'] grain-size `)`
972static ParseResult
973parseGrainsizeClause(OpAsmParser &parser, ClauseGrainsizeTypeAttr &grainsizeMod,
974 std::optional<OpAsmParser::UnresolvedOperand> &grainsize,
975 Type &grainsizeType) {
976 return parseGranularityClause<ClauseGrainsizeTypeAttr, ClauseGrainsizeType>(
977 parser, prescriptiveness&: grainsizeMod, operand&: grainsize, operandType&: grainsizeType,
978 symbolizeClause: &symbolizeClauseGrainsizeType, clauseName: "grainsize");
979}
980
981static void printGrainsizeClause(OpAsmPrinter &p, Operation *op,
982 ClauseGrainsizeTypeAttr grainsizeMod,
983 Value grainsize, mlir::Type grainsizeType) {
984 printGranularityClause<ClauseGrainsizeTypeAttr, ClauseGrainsizeType>(
985 p, op, prescriptiveness: grainsizeMod, operand: grainsize, operandType: grainsizeType,
986 stringifyClauseType: &stringifyClauseGrainsizeType);
987}
988
989//===----------------------------------------------------------------------===//
990// Parser and printer for num_tasks Clause
991//===----------------------------------------------------------------------===//
992
993// numtask ::= `num_tasks` `(` [strict ':'] num-tasks `)`
994static ParseResult
995parseNumTasksClause(OpAsmParser &parser, ClauseNumTasksTypeAttr &numTasksMod,
996 std::optional<OpAsmParser::UnresolvedOperand> &numTasks,
997 Type &numTasksType) {
998 return parseGranularityClause<ClauseNumTasksTypeAttr, ClauseNumTasksType>(
999 parser, prescriptiveness&: numTasksMod, operand&: numTasks, operandType&: numTasksType, symbolizeClause: &symbolizeClauseNumTasksType,
1000 clauseName: "num_tasks");
1001}
1002
1003static void printNumTasksClause(OpAsmPrinter &p, Operation *op,
1004 ClauseNumTasksTypeAttr numTasksMod,
1005 Value numTasks, mlir::Type numTasksType) {
1006 printGranularityClause<ClauseNumTasksTypeAttr, ClauseNumTasksType>(
1007 p, op, prescriptiveness: numTasksMod, operand: numTasks, operandType: numTasksType, stringifyClauseType: &stringifyClauseNumTasksType);
1008}
1009
1010//===----------------------------------------------------------------------===//
1011// Parser and printer for Heap Alloc Clause
1012//===----------------------------------------------------------------------===//
1013
1014/// operation ::= $in_type ( `(` $typeparams `)` )? ( `,` $shape )?
1015static ParseResult parseHeapAllocClause(
1016 OpAsmParser &parser, TypeAttr &inTypeAttr,
1017 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &typeparams,
1018 SmallVectorImpl<Type> &typeparamsTypes,
1019 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &shape,
1020 SmallVectorImpl<Type> &shapeTypes) {
1021 mlir::Type inType;
1022 if (parser.parseType(result&: inType))
1023 return mlir::failure();
1024 inTypeAttr = TypeAttr::get(type: inType);
1025
1026 if (!parser.parseOptionalLParen()) {
1027 // parse the LEN params of the derived type. (<params> : <types>)
1028 if (parser.parseOperandList(result&: typeparams, delimiter: OpAsmParser::Delimiter::None) ||
1029 parser.parseColonTypeList(result&: typeparamsTypes) || parser.parseRParen())
1030 return failure();
1031 }
1032
1033 if (!parser.parseOptionalComma()) {
1034 // parse size to scale by, vector of n dimensions of type index
1035 if (parser.parseOperandList(result&: shape, delimiter: OpAsmParser::Delimiter::None))
1036 return failure();
1037
1038 // TODO: This overrides the actual types of the operands, which might cause
1039 // issues when they don't match. At the moment this is done in place of
1040 // making the corresponding operand type `Variadic<Index>` because index
1041 // types are lowered to I64 prior to LLVM IR translation.
1042 shapeTypes.append(NumInputs: shape.size(), Elt: IndexType::get(context: parser.getContext()));
1043 }
1044
1045 return success();
1046}
1047
1048static void printHeapAllocClause(OpAsmPrinter &p, Operation *op,
1049 TypeAttr inType, ValueRange typeparams,
1050 TypeRange typeparamsTypes, ValueRange shape,
1051 TypeRange shapeTypes) {
1052 p << inType;
1053 if (!typeparams.empty()) {
1054 p << '(' << typeparams << " : " << typeparamsTypes << ')';
1055 }
1056 for (auto sh : shape) {
1057 p << ", ";
1058 p.printOperand(value: sh);
1059 }
1060}
1061
1062//===----------------------------------------------------------------------===//
1063// Parser, printer and verify for dyn_groupprivate Clause
1064//===----------------------------------------------------------------------===//
1065
1066static LogicalResult
1067verifyDynGroupprivateClause(Operation *op, AccessGroupModifierAttr accessGroup,
1068 FallbackModifierAttr fallback,
1069 Value dynGroupprivateSize) {
1070 if (!dynGroupprivateSize && (accessGroup || fallback))
1071 return op->emitOpError(message: "dyn_groupprivate modifiers require a size operand");
1072
1073 return success();
1074}
1075
1076static ParseResult parseDynGroupprivateClause(
1077 OpAsmParser &parser, AccessGroupModifierAttr &accessGroupAttr,
1078 FallbackModifierAttr &fallbackAttr,
1079 std::optional<OpAsmParser::UnresolvedOperand> &dynGroupprivateSize,
1080 Type &sizeType) {
1081
1082 bool parsedAccessGroup = false;
1083 bool parsedFallback = false;
1084 bool parsedSize = false;
1085
1086 return parser.parseCommaSeparatedList(parseElementFn: [&]() -> ParseResult {
1087 // Parse AccessGroupModifier.
1088 if (succeeded(Result: parser.parseOptionalKeyword(keyword: "cgroup"))) {
1089 if (parsedAccessGroup)
1090 return parser.emitError(loc: parser.getCurrentLocation(),
1091 message: "duplicate access group modifier");
1092 accessGroupAttr = AccessGroupModifierAttr::get(
1093 context: parser.getContext(), value: AccessGroupModifier::cgroup);
1094 parsedAccessGroup = true;
1095 return success();
1096 }
1097 // Parse FallbackModifier.
1098 if (succeeded(Result: parser.parseOptionalKeyword(keyword: "fallback"))) {
1099 if (parsedFallback)
1100 return parser.emitError(loc: parser.getCurrentLocation(),
1101 message: "duplicate fallback modifier");
1102 if (parser.parseLParen())
1103 return parser.emitError(loc: parser.getCurrentLocation(),
1104 message: "expected '(' after 'fallback'");
1105 llvm::StringRef fbKind;
1106 if (parser.parseKeyword(keyword: &fbKind))
1107 return parser.emitError(
1108 loc: parser.getCurrentLocation(),
1109 message: "expected fallback modifier (abort/null/default_mem)");
1110 std::optional<FallbackModifier> fbEnum;
1111 if (fbKind == "abort")
1112 fbEnum = FallbackModifier::abort;
1113 else if (fbKind == "null")
1114 fbEnum = FallbackModifier::null;
1115 else if (fbKind == "default_mem")
1116 fbEnum = FallbackModifier::default_mem;
1117 else
1118 return parser.emitError(loc: parser.getCurrentLocation(),
1119 message: "invalid fallback modifier '" + fbKind + "'");
1120 fallbackAttr = FallbackModifierAttr::get(context: parser.getContext(), value: *fbEnum);
1121 if (parser.parseRParen())
1122 return parser.emitError(loc: parser.getCurrentLocation(),
1123 message: "expected ')' after fallback modifier");
1124 parsedFallback = true;
1125 return success();
1126 }
1127 // Parse size operand.
1128 OpAsmParser::UnresolvedOperand operand;
1129 if (succeeded(Result: parser.parseOperand(result&: operand))) {
1130 if (parsedSize)
1131 return parser.emitError(loc: parser.getCurrentLocation(),
1132 message: "duplicate size operand");
1133 dynGroupprivateSize = operand;
1134 parsedSize = true;
1135 if (failed(Result: parser.parseColon()) || failed(Result: parser.parseType(result&: sizeType)))
1136 return parser.emitError(loc: parser.getCurrentLocation(),
1137 message: "expected ':' and type after size operand");
1138 return success();
1139 }
1140 return parser.emitError(loc: parser.getCurrentLocation(),
1141 message: "expected dyn_groupprivate_size operand");
1142 });
1143}
1144
1145static void printDynGroupprivateClause(OpAsmPrinter &printer, Operation *op,
1146 AccessGroupModifierAttr modifierFirst,
1147 FallbackModifierAttr modifierSecond,
1148 Value dynGroupprivateSize,
1149 Type sizeType) {
1150
1151 bool needsComma = false;
1152
1153 if (modifierFirst) {
1154 printer << modifierFirst.getValue();
1155 needsComma = true;
1156 }
1157
1158 if (modifierSecond) {
1159 if (needsComma)
1160 printer << ", ";
1161 printer << "fallback(";
1162 printer << modifierSecond.getValue();
1163 printer << ")";
1164 needsComma = true;
1165 }
1166
1167 if (dynGroupprivateSize) {
1168 if (needsComma)
1169 printer << ", ";
1170 printer << dynGroupprivateSize << " : " << sizeType;
1171 }
1172}
1173
1174//===----------------------------------------------------------------------===//
1175// Parser and printer for in_reduction Clause
1176//===----------------------------------------------------------------------===//
1177
1178/// Parses an `in_reduction` clause for an operation that does not give its
1179/// list items entry block arguments (e.g. `omp.target`). The expected format is
1180/// a comma-separated list of `[byref] @sym %var` followed by `: types`.
1181static ParseResult parseInReductionClause(
1182 OpAsmParser &parser,
1183 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &inReductionVars,
1184 SmallVectorImpl<Type> &inReductionTypes,
1185 DenseBoolArrayAttr &inReductionByref, ArrayAttr &inReductionSyms) {
1186 SmallVector<SymbolRefAttr> symbolVec;
1187 SmallVector<bool> isByRefVec;
1188
1189 if (parser.parseCommaSeparatedList(parseElementFn: [&]() {
1190 isByRefVec.push_back(Elt: parser.parseOptionalKeyword(keyword: "byref").succeeded());
1191 if (parser.parseAttribute(result&: symbolVec.emplace_back()) ||
1192 parser.parseOperand(result&: inReductionVars.emplace_back()))
1193 return failure();
1194 return success();
1195 }))
1196 return failure();
1197
1198 if (parser.parseColon())
1199 return failure();
1200
1201 if (parser.parseCommaSeparatedList(
1202 parseElementFn: [&]() { return parser.parseType(result&: inReductionTypes.emplace_back()); }))
1203 return failure();
1204
1205 if (inReductionVars.size() != inReductionTypes.size())
1206 return failure();
1207
1208 inReductionByref = makeDenseBoolArrayAttr(ctx: parser.getContext(), boolArray: isByRefVec);
1209 SmallVector<Attribute> symbolAttrs(symbolVec.begin(), symbolVec.end());
1210 inReductionSyms = ArrayAttr::get(context: parser.getContext(), value: symbolAttrs);
1211 return success();
1212}
1213
1214/// Prints an `in_reduction` clause for an operation that does not give its list
1215/// items entry block arguments (e.g. `omp.target`). Mirrors
1216/// `parseInReductionClause`.
1217static void printInReductionClause(OpAsmPrinter &p, Operation *op,
1218 ValueRange inReductionVars,
1219 TypeRange inReductionTypes,
1220 DenseBoolArrayAttr inReductionByref,
1221 ArrayAttr inReductionSyms) {
1222 MLIRContext *ctx = op->getContext();
1223
1224 ArrayAttr syms = inReductionSyms;
1225 if (!syms) {
1226 SmallVector<Attribute> values(inReductionVars.size(), nullptr);
1227 syms = ArrayAttr::get(context: ctx, value: values);
1228 }
1229
1230 DenseBoolArrayAttr byref = inReductionByref;
1231 if (!byref) {
1232 SmallVector<bool> values(inReductionVars.size(), false);
1233 byref = DenseBoolArrayAttr::get(context: ctx, content: values);
1234 }
1235
1236 llvm::interleaveComma(
1237 c: llvm::zip_equal(t&: inReductionVars, u: syms.getValue(), args: byref.asArrayRef()), os&: p,
1238 each_fn: [&p](auto t) {
1239 auto [var, sym, isByRef] = t;
1240 if (isByRef)
1241 p << "byref ";
1242 if (sym)
1243 p << sym << " ";
1244 p << var;
1245 });
1246 p << " : ";
1247 llvm::interleaveComma(c: inReductionTypes, os&: p);
1248}
1249
1250//===----------------------------------------------------------------------===//
1251// Parsers for operations including clauses that define entry block arguments.
1252//===----------------------------------------------------------------------===//
1253
1254namespace {
1255struct MapParseArgs {
1256 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars;
1257 SmallVectorImpl<Type> &types;
1258 MapParseArgs(SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars,
1259 SmallVectorImpl<Type> &types)
1260 : vars(vars), types(types) {}
1261};
1262struct PrivateParseArgs {
1263 llvm::SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars;
1264 llvm::SmallVectorImpl<Type> &types;
1265 ArrayAttr &syms;
1266 UnitAttr &needsBarrier;
1267 DenseI64ArrayAttr *mapIndices;
1268 PrivateParseArgs(SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars,
1269 SmallVectorImpl<Type> &types, ArrayAttr &syms,
1270 UnitAttr &needsBarrier,
1271 DenseI64ArrayAttr *mapIndices = nullptr)
1272 : vars(vars), types(types), syms(syms), needsBarrier(needsBarrier),
1273 mapIndices(mapIndices) {}
1274};
1275
1276struct LinearParseArgs {
1277 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars;
1278 SmallVectorImpl<Type> &types;
1279 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &stepVars;
1280 SmallVectorImpl<Type> &stepTypes;
1281 ArrayAttr &varTypes;
1282 ArrayAttr &modifiers;
1283 ArrayAttr &syms;
1284 LinearParseArgs(SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars,
1285 SmallVectorImpl<Type> &types,
1286 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &stepVars,
1287 SmallVectorImpl<Type> &stepTypes, ArrayAttr &varTypes,
1288 ArrayAttr &modifiers, ArrayAttr &syms)
1289 : vars(vars), types(types), stepVars(stepVars), stepTypes(stepTypes),
1290 varTypes(varTypes), modifiers(modifiers), syms(syms) {}
1291};
1292
1293struct ReductionParseArgs {
1294 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars;
1295 SmallVectorImpl<Type> &types;
1296 DenseBoolArrayAttr &byref;
1297 ArrayAttr &syms;
1298 ArrayAttr *modifier;
1299 ReductionParseArgs(SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars,
1300 SmallVectorImpl<Type> &types, DenseBoolArrayAttr &byref,
1301 ArrayAttr &syms, ArrayAttr *mod = nullptr)
1302 : vars(vars), types(types), byref(byref), syms(syms), modifier(mod) {}
1303};
1304
1305struct AllRegionParseArgs {
1306 std::optional<MapParseArgs> hasDeviceAddrArgs;
1307 std::optional<MapParseArgs> hostEvalArgs;
1308 std::optional<ReductionParseArgs> inReductionArgs;
1309 std::optional<LinearParseArgs> linearArgs;
1310 std::optional<MapParseArgs> mapArgs;
1311 std::optional<PrivateParseArgs> privateArgs;
1312 std::optional<ReductionParseArgs> reductionArgs;
1313 std::optional<ReductionParseArgs> taskReductionArgs;
1314 std::optional<MapParseArgs> useDeviceAddrArgs;
1315 std::optional<MapParseArgs> useDevicePtrArgs;
1316};
1317} // namespace
1318
1319static inline constexpr StringRef getPrivateNeedsBarrierSpelling() {
1320 return "private_barrier";
1321}
1322
1323static ParseResult parseClauseWithRegionArgs(
1324 OpAsmParser &parser,
1325 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &operands,
1326 SmallVectorImpl<Type> &types,
1327 SmallVectorImpl<OpAsmParser::Argument> &regionPrivateArgs,
1328 ArrayAttr *symbols = nullptr, DenseI64ArrayAttr *mapIndices = nullptr,
1329 DenseBoolArrayAttr *byref = nullptr, ArrayAttr *modifier = nullptr,
1330 UnitAttr *needsBarrier = nullptr) {
1331 SmallVector<SymbolRefAttr> symbolVec;
1332 SmallVector<int64_t> mapIndicesVec;
1333 SmallVector<bool> isByRefVec;
1334 unsigned regionArgOffset = regionPrivateArgs.size();
1335
1336 if (parser.parseLParen())
1337 return failure();
1338
1339 SmallVector<Attribute> modifiers;
1340 if (parser.parseCommaSeparatedList(parseElementFn: [&]() -> ParseResult {
1341 if (modifier) {
1342 ReductionModifier mod = ReductionModifier::defaultmod;
1343 if (succeeded(Result: parser.parseOptionalKeyword(keyword: "task")))
1344 mod = ReductionModifier::task;
1345 else if (succeeded(Result: parser.parseOptionalKeyword(keyword: "inscan")))
1346 mod = ReductionModifier::inscan;
1347 else if (succeeded(Result: parser.parseOptionalKeyword(keyword: "defaultmod")))
1348 mod = ReductionModifier::defaultmod;
1349 modifiers.push_back(
1350 Elt: ReductionModifierAttr::get(context: parser.getContext(), value: mod));
1351 }
1352 if (byref)
1353 isByRefVec.push_back(
1354 Elt: parser.parseOptionalKeyword(keyword: "byref").succeeded());
1355
1356 if (symbols && parser.parseAttribute(result&: symbolVec.emplace_back()))
1357 return failure();
1358
1359 if (parser.parseOperand(result&: operands.emplace_back()) ||
1360 parser.parseArrow() ||
1361 parser.parseArgument(result&: regionPrivateArgs.emplace_back()))
1362 return failure();
1363
1364 if (mapIndices) {
1365 if (parser.parseOptionalLSquare().succeeded()) {
1366 if (parser.parseKeyword(keyword: "map_idx") || parser.parseEqual() ||
1367 parser.parseInteger(result&: mapIndicesVec.emplace_back()) ||
1368 parser.parseRSquare())
1369 return failure();
1370 } else {
1371 mapIndicesVec.push_back(Elt: -1);
1372 }
1373 }
1374
1375 return success();
1376 }))
1377 return failure();
1378
1379 if (parser.parseColon() || parser.parseCommaSeparatedList(parseElementFn: [&]() {
1380 return parser.parseType(result&: types.emplace_back());
1381 }))
1382 return failure();
1383
1384 if (llvm::any_of(Range&: modifiers, P: [](Attribute attr) {
1385 return cast<ReductionModifierAttr>(Val&: attr).getValue() !=
1386 ReductionModifier::defaultmod;
1387 }))
1388 *modifier = ArrayAttr::get(context: parser.getContext(), value: modifiers);
1389
1390 if (operands.size() != types.size())
1391 return failure();
1392
1393 if (parser.parseRParen())
1394 return failure();
1395
1396 if (needsBarrier) {
1397 if (parser.parseOptionalKeyword(keyword: getPrivateNeedsBarrierSpelling())
1398 .succeeded())
1399 *needsBarrier = mlir::UnitAttr::get(context: parser.getContext());
1400 }
1401
1402 auto *argsBegin = regionPrivateArgs.begin();
1403 MutableArrayRef argsSubrange(argsBegin + regionArgOffset,
1404 argsBegin + regionArgOffset + types.size());
1405 for (auto [prv, type] : llvm::zip_equal(t&: argsSubrange, u&: types)) {
1406 prv.type = type;
1407 }
1408
1409 if (symbols) {
1410 SmallVector<Attribute> symbolAttrs(symbolVec.begin(), symbolVec.end());
1411 *symbols = ArrayAttr::get(context: parser.getContext(), value: symbolAttrs);
1412 }
1413
1414 if (!mapIndicesVec.empty())
1415 *mapIndices =
1416 mlir::DenseI64ArrayAttr::get(context: parser.getContext(), content: mapIndicesVec);
1417
1418 if (byref)
1419 *byref = makeDenseBoolArrayAttr(ctx: parser.getContext(), boolArray: isByRefVec);
1420
1421 return success();
1422}
1423
1424static ParseResult parseBlockArgClause(
1425 OpAsmParser &parser,
1426 llvm::SmallVectorImpl<OpAsmParser::Argument> &entryBlockArgs,
1427 StringRef keyword, std::optional<MapParseArgs> mapArgs) {
1428 if (succeeded(Result: parser.parseOptionalKeyword(keyword))) {
1429 if (!mapArgs)
1430 return failure();
1431
1432 if (failed(Result: parseClauseWithRegionArgs(parser, operands&: mapArgs->vars, types&: mapArgs->types,
1433 regionPrivateArgs&: entryBlockArgs)))
1434 return failure();
1435 }
1436 return success();
1437}
1438
1439static ParseResult parseBlockArgClause(
1440 OpAsmParser &parser,
1441 llvm::SmallVectorImpl<OpAsmParser::Argument> &entryBlockArgs,
1442 StringRef keyword, std::optional<PrivateParseArgs> privateArgs) {
1443 if (succeeded(Result: parser.parseOptionalKeyword(keyword))) {
1444 if (!privateArgs)
1445 return failure();
1446
1447 if (failed(Result: parseClauseWithRegionArgs(
1448 parser, operands&: privateArgs->vars, types&: privateArgs->types, regionPrivateArgs&: entryBlockArgs,
1449 symbols: &privateArgs->syms, mapIndices: privateArgs->mapIndices, /*byref=*/nullptr,
1450 /*modifier=*/nullptr, needsBarrier: &privateArgs->needsBarrier)))
1451 return failure();
1452 }
1453 return success();
1454}
1455
1456static ParseResult parseBlockArgClause(
1457 OpAsmParser &parser,
1458 llvm::SmallVectorImpl<OpAsmParser::Argument> &entryBlockArgs,
1459 StringRef keyword, std::optional<ReductionParseArgs> reductionArgs) {
1460 if (succeeded(Result: parser.parseOptionalKeyword(keyword))) {
1461 if (!reductionArgs)
1462 return failure();
1463 if (failed(Result: parseClauseWithRegionArgs(
1464 parser, operands&: reductionArgs->vars, types&: reductionArgs->types, regionPrivateArgs&: entryBlockArgs,
1465 symbols: &reductionArgs->syms, /*mapIndices=*/nullptr, byref: &reductionArgs->byref,
1466 modifier: reductionArgs->modifier)))
1467 return failure();
1468 }
1469 return success();
1470}
1471
1472/// linear-entry := [modifier `(`] symbol ssa-id `->` ssa-id `:` type `=`
1473/// ssa-id `:` type [`)`]
1474static ParseResult parseBlockArgClause(
1475 OpAsmParser &parser,
1476 llvm::SmallVectorImpl<OpAsmParser::Argument> &entryBlockArgs,
1477 StringRef keyword, std::optional<LinearParseArgs> linearArgs) {
1478 if (!linearArgs || failed(Result: parser.parseOptionalKeyword(keyword)))
1479 return success();
1480
1481 MLIRContext *ctx = parser.getContext();
1482 SmallVector<Attribute> syms, modifiers;
1483 if (parser.parseLParen() ||
1484 parser.parseCommaSeparatedList(parseElementFn: [&]() -> ParseResult {
1485 std::optional<LinearModifier> modifier;
1486 if (succeeded(Result: parser.parseOptionalKeyword(keyword: "val")))
1487 modifier = LinearModifier::val;
1488 else if (succeeded(Result: parser.parseOptionalKeyword(keyword: "ref")))
1489 modifier = LinearModifier::ref;
1490 else if (succeeded(Result: parser.parseOptionalKeyword(keyword: "uval")))
1491 modifier = LinearModifier::uval;
1492 if (modifier && parser.parseLParen())
1493 return failure();
1494
1495 SymbolRefAttr sym;
1496 OpAsmParser::Argument &arg = entryBlockArgs.emplace_back();
1497 if (parser.parseAttribute(result&: sym) ||
1498 parser.parseOperand(result&: linearArgs->vars.emplace_back()) ||
1499 parser.parseArrow() || parser.parseArgument(result&: arg) ||
1500 parser.parseColonType(result&: linearArgs->types.emplace_back()) ||
1501 parser.parseEqual() ||
1502 parser.parseOperand(result&: linearArgs->stepVars.emplace_back()) ||
1503 parser.parseColonType(result&: linearArgs->stepTypes.emplace_back()))
1504 return failure();
1505 arg.type = linearArgs->types.back();
1506
1507 if (modifier && parser.parseRParen())
1508 return failure();
1509
1510 syms.push_back(Elt: sym);
1511 modifiers.push_back(
1512 Elt: modifier ? Attribute(LinearModifierAttr::get(context: ctx, value: *modifier))
1513 : Attribute(UnitAttr::get(context: ctx)));
1514 return success();
1515 }) ||
1516 parser.parseRParen())
1517 return failure();
1518
1519 linearArgs->syms = ArrayAttr::get(context: ctx, value: syms);
1520 linearArgs->modifiers = ArrayAttr::get(context: ctx, value: modifiers);
1521
1522 if (succeeded(Result: parser.parseOptionalKeyword(keyword: "linear_var_types")) &&
1523 (parser.parseLParen() || parser.parseAttribute(result&: linearArgs->varTypes) ||
1524 parser.parseRParen()))
1525 return failure();
1526 return success();
1527}
1528
1529static ParseResult parseBlockArgRegion(OpAsmParser &parser, Region &region,
1530 AllRegionParseArgs args) {
1531 llvm::SmallVector<OpAsmParser::Argument> entryBlockArgs;
1532
1533 if (failed(Result: parseBlockArgClause(parser, entryBlockArgs, keyword: "has_device_addr",
1534 mapArgs: args.hasDeviceAddrArgs)))
1535 return parser.emitError(loc: parser.getCurrentLocation())
1536 << "invalid `has_device_addr` format";
1537
1538 if (failed(Result: parseBlockArgClause(parser, entryBlockArgs, keyword: "host_eval",
1539 mapArgs: args.hostEvalArgs)))
1540 return parser.emitError(loc: parser.getCurrentLocation())
1541 << "invalid `host_eval` format";
1542
1543 if (failed(Result: parseBlockArgClause(parser, entryBlockArgs, keyword: "in_reduction",
1544 reductionArgs: args.inReductionArgs)))
1545 return parser.emitError(loc: parser.getCurrentLocation())
1546 << "invalid `in_reduction` format";
1547
1548 if (failed(Result: parseBlockArgClause(parser, entryBlockArgs, keyword: "linear",
1549 linearArgs: args.linearArgs)))
1550 return parser.emitError(loc: parser.getCurrentLocation())
1551 << "invalid `linear` format";
1552
1553 if (failed(Result: parseBlockArgClause(parser, entryBlockArgs, keyword: "map_entries",
1554 mapArgs: args.mapArgs)))
1555 return parser.emitError(loc: parser.getCurrentLocation())
1556 << "invalid `map_entries` format";
1557
1558 if (failed(Result: parseBlockArgClause(parser, entryBlockArgs, keyword: "private",
1559 privateArgs: args.privateArgs)))
1560 return parser.emitError(loc: parser.getCurrentLocation())
1561 << "invalid `private` format";
1562
1563 if (failed(Result: parseBlockArgClause(parser, entryBlockArgs, keyword: "reduction",
1564 reductionArgs: args.reductionArgs)))
1565 return parser.emitError(loc: parser.getCurrentLocation())
1566 << "invalid `reduction` format";
1567
1568 if (failed(Result: parseBlockArgClause(parser, entryBlockArgs, keyword: "task_reduction",
1569 reductionArgs: args.taskReductionArgs)))
1570 return parser.emitError(loc: parser.getCurrentLocation())
1571 << "invalid `task_reduction` format";
1572
1573 if (failed(Result: parseBlockArgClause(parser, entryBlockArgs, keyword: "use_device_addr",
1574 mapArgs: args.useDeviceAddrArgs)))
1575 return parser.emitError(loc: parser.getCurrentLocation())
1576 << "invalid `use_device_addr` format";
1577
1578 if (failed(Result: parseBlockArgClause(parser, entryBlockArgs, keyword: "use_device_ptr",
1579 mapArgs: args.useDevicePtrArgs)))
1580 return parser.emitError(loc: parser.getCurrentLocation())
1581 << "invalid `use_device_addr` format";
1582
1583 return parser.parseRegion(region, arguments: entryBlockArgs);
1584}
1585
1586// These parseXyz functions correspond to the custom<Xyz> definitions
1587// in the .td file(s).
1588static ParseResult parseTargetOpRegion(
1589 OpAsmParser &parser, Region &region,
1590 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &hasDeviceAddrVars,
1591 SmallVectorImpl<Type> &hasDeviceAddrTypes,
1592 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &hostEvalVars,
1593 SmallVectorImpl<Type> &hostEvalTypes,
1594 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &mapVars,
1595 SmallVectorImpl<Type> &mapTypes,
1596 llvm::SmallVectorImpl<OpAsmParser::UnresolvedOperand> &privateVars,
1597 llvm::SmallVectorImpl<Type> &privateTypes, ArrayAttr &privateSyms,
1598 UnitAttr &privateNeedsBarrier, DenseI64ArrayAttr &privateMaps) {
1599 AllRegionParseArgs args;
1600 args.hasDeviceAddrArgs.emplace(args&: hasDeviceAddrVars, args&: hasDeviceAddrTypes);
1601 args.hostEvalArgs.emplace(args&: hostEvalVars, args&: hostEvalTypes);
1602 args.mapArgs.emplace(args&: mapVars, args&: mapTypes);
1603 args.privateArgs.emplace(args&: privateVars, args&: privateTypes, args&: privateSyms,
1604 args&: privateNeedsBarrier, args: &privateMaps);
1605 return parseBlockArgRegion(parser, region, args);
1606}
1607
1608static ParseResult parseInReductionPrivateRegion(
1609 OpAsmParser &parser, Region &region,
1610 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &inReductionVars,
1611 SmallVectorImpl<Type> &inReductionTypes,
1612 DenseBoolArrayAttr &inReductionByref, ArrayAttr &inReductionSyms,
1613 llvm::SmallVectorImpl<OpAsmParser::UnresolvedOperand> &privateVars,
1614 llvm::SmallVectorImpl<Type> &privateTypes, ArrayAttr &privateSyms,
1615 UnitAttr &privateNeedsBarrier) {
1616 AllRegionParseArgs args;
1617 args.inReductionArgs.emplace(args&: inReductionVars, args&: inReductionTypes,
1618 args&: inReductionByref, args&: inReductionSyms);
1619 args.privateArgs.emplace(args&: privateVars, args&: privateTypes, args&: privateSyms,
1620 args&: privateNeedsBarrier);
1621 return parseBlockArgRegion(parser, region, args);
1622}
1623
1624static ParseResult parseInReductionPrivateReductionRegion(
1625 OpAsmParser &parser, Region &region,
1626 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &inReductionVars,
1627 SmallVectorImpl<Type> &inReductionTypes,
1628 DenseBoolArrayAttr &inReductionByref, ArrayAttr &inReductionSyms,
1629 llvm::SmallVectorImpl<OpAsmParser::UnresolvedOperand> &privateVars,
1630 llvm::SmallVectorImpl<Type> &privateTypes, ArrayAttr &privateSyms,
1631 UnitAttr &privateNeedsBarrier, ArrayAttr &reductionMods,
1632 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &reductionVars,
1633 SmallVectorImpl<Type> &reductionTypes, DenseBoolArrayAttr &reductionByref,
1634 ArrayAttr &reductionSyms) {
1635 AllRegionParseArgs args;
1636 args.inReductionArgs.emplace(args&: inReductionVars, args&: inReductionTypes,
1637 args&: inReductionByref, args&: inReductionSyms);
1638 args.privateArgs.emplace(args&: privateVars, args&: privateTypes, args&: privateSyms,
1639 args&: privateNeedsBarrier);
1640 args.reductionArgs.emplace(args&: reductionVars, args&: reductionTypes, args&: reductionByref,
1641 args&: reductionSyms, args: &reductionMods);
1642 return parseBlockArgRegion(parser, region, args);
1643}
1644
1645static ParseResult parsePrivateRegion(
1646 OpAsmParser &parser, Region &region,
1647 llvm::SmallVectorImpl<OpAsmParser::UnresolvedOperand> &privateVars,
1648 llvm::SmallVectorImpl<Type> &privateTypes, ArrayAttr &privateSyms,
1649 UnitAttr &privateNeedsBarrier) {
1650 AllRegionParseArgs args;
1651 args.privateArgs.emplace(args&: privateVars, args&: privateTypes, args&: privateSyms,
1652 args&: privateNeedsBarrier);
1653 return parseBlockArgRegion(parser, region, args);
1654}
1655
1656static ParseResult parsePrivateReductionRegion(
1657 OpAsmParser &parser, Region &region,
1658 llvm::SmallVectorImpl<OpAsmParser::UnresolvedOperand> &privateVars,
1659 llvm::SmallVectorImpl<Type> &privateTypes, ArrayAttr &privateSyms,
1660 UnitAttr &privateNeedsBarrier, ArrayAttr &reductionMods,
1661 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &reductionVars,
1662 SmallVectorImpl<Type> &reductionTypes, DenseBoolArrayAttr &reductionByref,
1663 ArrayAttr &reductionSyms) {
1664 AllRegionParseArgs args;
1665 args.privateArgs.emplace(args&: privateVars, args&: privateTypes, args&: privateSyms,
1666 args&: privateNeedsBarrier);
1667 args.reductionArgs.emplace(args&: reductionVars, args&: reductionTypes, args&: reductionByref,
1668 args&: reductionSyms, args: &reductionMods);
1669 return parseBlockArgRegion(parser, region, args);
1670}
1671
1672static ParseResult parseLinearPrivateReductionRegion(
1673 OpAsmParser &parser, Region &region,
1674 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &linearVars,
1675 SmallVectorImpl<Type> &linearTypes,
1676 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &linearStepVars,
1677 SmallVectorImpl<Type> &linearStepTypes, ArrayAttr &linearVarTypes,
1678 ArrayAttr &linearModifiers, ArrayAttr &linearSyms,
1679 llvm::SmallVectorImpl<OpAsmParser::UnresolvedOperand> &privateVars,
1680 llvm::SmallVectorImpl<Type> &privateTypes, ArrayAttr &privateSyms,
1681 UnitAttr &privateNeedsBarrier, ArrayAttr &reductionMods,
1682 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &reductionVars,
1683 SmallVectorImpl<Type> &reductionTypes, DenseBoolArrayAttr &reductionByref,
1684 ArrayAttr &reductionSyms) {
1685 AllRegionParseArgs args;
1686 args.linearArgs.emplace(args&: linearVars, args&: linearTypes, args&: linearStepVars,
1687 args&: linearStepTypes, args&: linearVarTypes, args&: linearModifiers,
1688 args&: linearSyms);
1689 args.privateArgs.emplace(args&: privateVars, args&: privateTypes, args&: privateSyms,
1690 args&: privateNeedsBarrier);
1691 args.reductionArgs.emplace(args&: reductionVars, args&: reductionTypes, args&: reductionByref,
1692 args&: reductionSyms, args: &reductionMods);
1693 return parseBlockArgRegion(parser, region, args);
1694}
1695
1696static ParseResult parseTaskReductionRegion(
1697 OpAsmParser &parser, Region &region,
1698 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &taskReductionVars,
1699 SmallVectorImpl<Type> &taskReductionTypes,
1700 DenseBoolArrayAttr &taskReductionByref, ArrayAttr &taskReductionSyms) {
1701 AllRegionParseArgs args;
1702 args.taskReductionArgs.emplace(args&: taskReductionVars, args&: taskReductionTypes,
1703 args&: taskReductionByref, args&: taskReductionSyms);
1704 return parseBlockArgRegion(parser, region, args);
1705}
1706
1707static ParseResult parseUseDeviceAddrUseDevicePtrRegion(
1708 OpAsmParser &parser, Region &region,
1709 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &useDeviceAddrVars,
1710 SmallVectorImpl<Type> &useDeviceAddrTypes,
1711 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &useDevicePtrVars,
1712 SmallVectorImpl<Type> &useDevicePtrTypes) {
1713 AllRegionParseArgs args;
1714 args.useDeviceAddrArgs.emplace(args&: useDeviceAddrVars, args&: useDeviceAddrTypes);
1715 args.useDevicePtrArgs.emplace(args&: useDevicePtrVars, args&: useDevicePtrTypes);
1716 return parseBlockArgRegion(parser, region, args);
1717}
1718
1719//===----------------------------------------------------------------------===//
1720// Printers for operations including clauses that define entry block arguments.
1721//===----------------------------------------------------------------------===//
1722
1723namespace {
1724struct MapPrintArgs {
1725 ValueRange vars;
1726 TypeRange types;
1727 MapPrintArgs(ValueRange vars, TypeRange types) : vars(vars), types(types) {}
1728};
1729struct PrivatePrintArgs {
1730 ValueRange vars;
1731 TypeRange types;
1732 ArrayAttr syms;
1733 UnitAttr needsBarrier;
1734 DenseI64ArrayAttr mapIndices;
1735 PrivatePrintArgs(ValueRange vars, TypeRange types, ArrayAttr syms,
1736 UnitAttr needsBarrier, DenseI64ArrayAttr mapIndices)
1737 : vars(vars), types(types), syms(syms), needsBarrier(needsBarrier),
1738 mapIndices(mapIndices) {}
1739};
1740struct LinearPrintArgs {
1741 ValueRange vars;
1742 TypeRange types;
1743 ValueRange stepVars;
1744 TypeRange stepTypes;
1745 ArrayAttr varTypes;
1746 ArrayAttr modifiers;
1747 ArrayAttr syms;
1748 LinearPrintArgs(ValueRange vars, TypeRange types, ValueRange stepVars,
1749 TypeRange stepTypes, ArrayAttr varTypes, ArrayAttr modifiers,
1750 ArrayAttr syms)
1751 : vars(vars), types(types), stepVars(stepVars), stepTypes(stepTypes),
1752 varTypes(varTypes), modifiers(modifiers), syms(syms) {}
1753};
1754struct ReductionPrintArgs {
1755 ValueRange vars;
1756 TypeRange types;
1757 DenseBoolArrayAttr byref;
1758 ArrayAttr syms;
1759 ArrayAttr modifier;
1760 ReductionPrintArgs(ValueRange vars, TypeRange types, DenseBoolArrayAttr byref,
1761 ArrayAttr syms, ArrayAttr mod = nullptr)
1762 : vars(vars), types(types), byref(byref), syms(syms), modifier(mod) {}
1763};
1764struct AllRegionPrintArgs {
1765 std::optional<MapPrintArgs> hasDeviceAddrArgs;
1766 std::optional<MapPrintArgs> hostEvalArgs;
1767 std::optional<ReductionPrintArgs> inReductionArgs;
1768 std::optional<LinearPrintArgs> linearArgs;
1769 std::optional<MapPrintArgs> mapArgs;
1770 std::optional<PrivatePrintArgs> privateArgs;
1771 std::optional<ReductionPrintArgs> reductionArgs;
1772 std::optional<ReductionPrintArgs> taskReductionArgs;
1773 std::optional<MapPrintArgs> useDeviceAddrArgs;
1774 std::optional<MapPrintArgs> useDevicePtrArgs;
1775};
1776} // namespace
1777
1778static void printClauseWithRegionArgs(
1779 OpAsmPrinter &p, MLIRContext *ctx, StringRef clauseName,
1780 ValueRange argsSubrange, ValueRange operands, TypeRange types,
1781 ArrayAttr symbols = nullptr, DenseI64ArrayAttr mapIndices = nullptr,
1782 DenseBoolArrayAttr byref = nullptr, ArrayAttr modifier = nullptr,
1783 UnitAttr needsBarrier = nullptr) {
1784 if (argsSubrange.empty())
1785 return;
1786
1787 p << clauseName << "(";
1788
1789 if (!symbols) {
1790 llvm::SmallVector<Attribute> values(operands.size(), nullptr);
1791 symbols = ArrayAttr::get(context: ctx, value: values);
1792 }
1793
1794 if (!mapIndices) {
1795 llvm::SmallVector<int64_t> values(operands.size(), -1);
1796 mapIndices = DenseI64ArrayAttr::get(context: ctx, content: values);
1797 }
1798
1799 if (!byref) {
1800 mlir::SmallVector<bool> values(operands.size(), false);
1801 byref = DenseBoolArrayAttr::get(context: ctx, content: values);
1802 }
1803
1804 unsigned index = 0;
1805 llvm::interleaveComma(
1806 c: llvm::zip_equal(t&: operands, u&: argsSubrange, args&: symbols, args: mapIndices.asArrayRef(),
1807 args: byref.asArrayRef()),
1808 os&: p, each_fn: [&](auto t) {
1809 auto [op, arg, sym, map, isByRef] = t;
1810 if (modifier) {
1811 ReductionModifier mod =
1812 cast<ReductionModifierAttr>(Val: modifier[index]).getValue();
1813 if (mod != ReductionModifier::defaultmod)
1814 p << stringifyReductionModifier(val: mod) << " ";
1815 }
1816 ++index;
1817 if (isByRef)
1818 p << "byref ";
1819 if (sym)
1820 p << sym << " ";
1821 p << op << " -> " << arg;
1822 if (map != -1)
1823 p << " [map_idx=" << map << "]";
1824 });
1825 p << " : ";
1826 llvm::interleaveComma(c: types, os&: p);
1827 p << ") ";
1828
1829 if (needsBarrier)
1830 p << getPrivateNeedsBarrierSpelling() << " ";
1831}
1832
1833static void printBlockArgClause(OpAsmPrinter &p, MLIRContext *ctx,
1834 StringRef clauseName, ValueRange argsSubrange,
1835 std::optional<MapPrintArgs> mapArgs) {
1836 if (mapArgs)
1837 printClauseWithRegionArgs(p, ctx, clauseName, argsSubrange, operands: mapArgs->vars,
1838 types: mapArgs->types);
1839}
1840
1841static void printBlockArgClause(OpAsmPrinter &p, MLIRContext *ctx,
1842 StringRef clauseName, ValueRange argsSubrange,
1843 std::optional<PrivatePrintArgs> privateArgs) {
1844 if (privateArgs)
1845 printClauseWithRegionArgs(
1846 p, ctx, clauseName, argsSubrange, operands: privateArgs->vars, types: privateArgs->types,
1847 symbols: privateArgs->syms, mapIndices: privateArgs->mapIndices, /*byref=*/nullptr,
1848 /*modifier=*/nullptr, needsBarrier: privateArgs->needsBarrier);
1849}
1850
1851static void
1852printBlockArgClause(OpAsmPrinter &p, MLIRContext *ctx, StringRef clauseName,
1853 ValueRange argsSubrange,
1854 std::optional<ReductionPrintArgs> reductionArgs) {
1855 if (reductionArgs)
1856 printClauseWithRegionArgs(p, ctx, clauseName, argsSubrange,
1857 operands: reductionArgs->vars, types: reductionArgs->types,
1858 symbols: reductionArgs->syms, /*mapIndices=*/nullptr,
1859 byref: reductionArgs->byref, modifier: reductionArgs->modifier);
1860}
1861
1862static void printBlockArgClause(OpAsmPrinter &p, MLIRContext *ctx,
1863 StringRef clauseName, ValueRange argsSubrange,
1864 std::optional<LinearPrintArgs> linearArgs) {
1865 if (!linearArgs || argsSubrange.empty())
1866 return;
1867
1868 p << clauseName << "(";
1869 for (auto [i, var] : llvm::enumerate(First&: linearArgs->vars)) {
1870 if (i != 0)
1871 p << ", ";
1872 auto modifier =
1873 linearArgs->modifiers
1874 ? dyn_cast_if_present<LinearModifierAttr>(Val: linearArgs->modifiers[i])
1875 : nullptr;
1876 if (modifier)
1877 p << stringifyLinearModifier(val: modifier.getValue()) << "(";
1878 if (linearArgs->syms)
1879 p << linearArgs->syms[i] << " ";
1880 p << var << " -> " << argsSubrange[i] << " : " << linearArgs->types[i]
1881 << " = " << linearArgs->stepVars[i] << " : " << linearArgs->stepTypes[i];
1882 if (modifier)
1883 p << ")";
1884 }
1885 p << ") ";
1886
1887 if (linearArgs->varTypes)
1888 p << "linear_var_types(" << linearArgs->varTypes << ") ";
1889}
1890
1891static void printBlockArgRegion(OpAsmPrinter &p, Operation *op, Region &region,
1892 const AllRegionPrintArgs &args) {
1893 auto iface = llvm::cast<mlir::omp::BlockArgOpenMPOpInterface>(Val: op);
1894 MLIRContext *ctx = op->getContext();
1895
1896 printBlockArgClause(p, ctx, clauseName: "has_device_addr",
1897 argsSubrange: iface.getHasDeviceAddrBlockArgs(),
1898 mapArgs: args.hasDeviceAddrArgs);
1899 printBlockArgClause(p, ctx, clauseName: "host_eval", argsSubrange: iface.getHostEvalBlockArgs(),
1900 mapArgs: args.hostEvalArgs);
1901 printBlockArgClause(p, ctx, clauseName: "in_reduction", argsSubrange: iface.getInReductionBlockArgs(),
1902 reductionArgs: args.inReductionArgs);
1903 printBlockArgClause(p, ctx, clauseName: "linear", argsSubrange: iface.getLinearBlockArgs(),
1904 linearArgs: args.linearArgs);
1905 printBlockArgClause(p, ctx, clauseName: "map_entries", argsSubrange: iface.getMapBlockArgs(),
1906 mapArgs: args.mapArgs);
1907 printBlockArgClause(p, ctx, clauseName: "private", argsSubrange: iface.getPrivateBlockArgs(),
1908 privateArgs: args.privateArgs);
1909 printBlockArgClause(p, ctx, clauseName: "reduction", argsSubrange: iface.getReductionBlockArgs(),
1910 reductionArgs: args.reductionArgs);
1911 printBlockArgClause(p, ctx, clauseName: "task_reduction",
1912 argsSubrange: iface.getTaskReductionBlockArgs(),
1913 reductionArgs: args.taskReductionArgs);
1914 printBlockArgClause(p, ctx, clauseName: "use_device_addr",
1915 argsSubrange: iface.getUseDeviceAddrBlockArgs(),
1916 mapArgs: args.useDeviceAddrArgs);
1917 printBlockArgClause(p, ctx, clauseName: "use_device_ptr",
1918 argsSubrange: iface.getUseDevicePtrBlockArgs(), mapArgs: args.useDevicePtrArgs);
1919
1920 p.printRegion(blocks&: region, /*printEntryBlockArgs=*/false);
1921}
1922
1923// These parseXyz functions correspond to the custom<Xyz> definitions
1924// in the .td file(s).
1925static void printTargetOpRegion(OpAsmPrinter &p, Operation *op, Region &region,
1926 ValueRange hasDeviceAddrVars,
1927 TypeRange hasDeviceAddrTypes,
1928 ValueRange hostEvalVars,
1929 TypeRange hostEvalTypes, ValueRange mapVars,
1930 TypeRange mapTypes, ValueRange privateVars,
1931 TypeRange privateTypes, ArrayAttr privateSyms,
1932 UnitAttr privateNeedsBarrier,
1933 DenseI64ArrayAttr privateMaps) {
1934 AllRegionPrintArgs args;
1935 args.hasDeviceAddrArgs.emplace(args&: hasDeviceAddrVars, args&: hasDeviceAddrTypes);
1936 args.hostEvalArgs.emplace(args&: hostEvalVars, args&: hostEvalTypes);
1937 args.mapArgs.emplace(args&: mapVars, args&: mapTypes);
1938 args.privateArgs.emplace(args&: privateVars, args&: privateTypes, args&: privateSyms,
1939 args&: privateNeedsBarrier, args&: privateMaps);
1940 printBlockArgRegion(p, op, region, args);
1941}
1942
1943static void printInReductionPrivateRegion(
1944 OpAsmPrinter &p, Operation *op, Region &region, ValueRange inReductionVars,
1945 TypeRange inReductionTypes, DenseBoolArrayAttr inReductionByref,
1946 ArrayAttr inReductionSyms, ValueRange privateVars, TypeRange privateTypes,
1947 ArrayAttr privateSyms, UnitAttr privateNeedsBarrier) {
1948 AllRegionPrintArgs args;
1949 args.inReductionArgs.emplace(args&: inReductionVars, args&: inReductionTypes,
1950 args&: inReductionByref, args&: inReductionSyms);
1951 args.privateArgs.emplace(args&: privateVars, args&: privateTypes, args&: privateSyms,
1952 args&: privateNeedsBarrier,
1953 /*mapIndices=*/args: nullptr);
1954 printBlockArgRegion(p, op, region, args);
1955}
1956
1957static void printInReductionPrivateReductionRegion(
1958 OpAsmPrinter &p, Operation *op, Region &region, ValueRange inReductionVars,
1959 TypeRange inReductionTypes, DenseBoolArrayAttr inReductionByref,
1960 ArrayAttr inReductionSyms, ValueRange privateVars, TypeRange privateTypes,
1961 ArrayAttr privateSyms, UnitAttr privateNeedsBarrier,
1962 ArrayAttr reductionMods, ValueRange reductionVars, TypeRange reductionTypes,
1963 DenseBoolArrayAttr reductionByref, ArrayAttr reductionSyms) {
1964 AllRegionPrintArgs args;
1965 args.inReductionArgs.emplace(args&: inReductionVars, args&: inReductionTypes,
1966 args&: inReductionByref, args&: inReductionSyms);
1967 args.privateArgs.emplace(args&: privateVars, args&: privateTypes, args&: privateSyms,
1968 args&: privateNeedsBarrier,
1969 /*mapIndices=*/args: nullptr);
1970 args.reductionArgs.emplace(args&: reductionVars, args&: reductionTypes, args&: reductionByref,
1971 args&: reductionSyms, args&: reductionMods);
1972 printBlockArgRegion(p, op, region, args);
1973}
1974
1975static void printPrivateRegion(OpAsmPrinter &p, Operation *op, Region &region,
1976 ValueRange privateVars, TypeRange privateTypes,
1977 ArrayAttr privateSyms,
1978 UnitAttr privateNeedsBarrier) {
1979 AllRegionPrintArgs args;
1980 args.privateArgs.emplace(args&: privateVars, args&: privateTypes, args&: privateSyms,
1981 args&: privateNeedsBarrier,
1982 /*mapIndices=*/args: nullptr);
1983 printBlockArgRegion(p, op, region, args);
1984}
1985
1986static void printPrivateReductionRegion(
1987 OpAsmPrinter &p, Operation *op, Region &region, ValueRange privateVars,
1988 TypeRange privateTypes, ArrayAttr privateSyms, UnitAttr privateNeedsBarrier,
1989 ArrayAttr reductionMods, ValueRange reductionVars, TypeRange reductionTypes,
1990 DenseBoolArrayAttr reductionByref, ArrayAttr reductionSyms) {
1991 AllRegionPrintArgs args;
1992 args.privateArgs.emplace(args&: privateVars, args&: privateTypes, args&: privateSyms,
1993 args&: privateNeedsBarrier,
1994 /*mapIndices=*/args: nullptr);
1995 args.reductionArgs.emplace(args&: reductionVars, args&: reductionTypes, args&: reductionByref,
1996 args&: reductionSyms, args&: reductionMods);
1997 printBlockArgRegion(p, op, region, args);
1998}
1999
2000static void printLinearPrivateReductionRegion(
2001 OpAsmPrinter &p, Operation *op, Region &region, ValueRange linearVars,
2002 TypeRange linearTypes, ValueRange linearStepVars, TypeRange linearStepTypes,
2003 ArrayAttr linearVarTypes, ArrayAttr linearModifiers, ArrayAttr linearSyms,
2004 ValueRange privateVars, TypeRange privateTypes, ArrayAttr privateSyms,
2005 UnitAttr privateNeedsBarrier, ArrayAttr reductionMods,
2006 ValueRange reductionVars, TypeRange reductionTypes,
2007 DenseBoolArrayAttr reductionByref, ArrayAttr reductionSyms) {
2008 AllRegionPrintArgs args;
2009 args.linearArgs.emplace(args&: linearVars, args&: linearTypes, args&: linearStepVars,
2010 args&: linearStepTypes, args&: linearVarTypes, args&: linearModifiers,
2011 args&: linearSyms);
2012 args.privateArgs.emplace(args&: privateVars, args&: privateTypes, args&: privateSyms,
2013 args&: privateNeedsBarrier,
2014 /*mapIndices=*/args: nullptr);
2015 args.reductionArgs.emplace(args&: reductionVars, args&: reductionTypes, args&: reductionByref,
2016 args&: reductionSyms, args&: reductionMods);
2017 printBlockArgRegion(p, op, region, args);
2018}
2019
2020static void printTaskReductionRegion(OpAsmPrinter &p, Operation *op,
2021 Region &region,
2022 ValueRange taskReductionVars,
2023 TypeRange taskReductionTypes,
2024 DenseBoolArrayAttr taskReductionByref,
2025 ArrayAttr taskReductionSyms) {
2026 AllRegionPrintArgs args;
2027 args.taskReductionArgs.emplace(args&: taskReductionVars, args&: taskReductionTypes,
2028 args&: taskReductionByref, args&: taskReductionSyms);
2029 printBlockArgRegion(p, op, region, args);
2030}
2031
2032static void printUseDeviceAddrUseDevicePtrRegion(OpAsmPrinter &p, Operation *op,
2033 Region &region,
2034 ValueRange useDeviceAddrVars,
2035 TypeRange useDeviceAddrTypes,
2036 ValueRange useDevicePtrVars,
2037 TypeRange useDevicePtrTypes) {
2038 AllRegionPrintArgs args;
2039 args.useDeviceAddrArgs.emplace(args&: useDeviceAddrVars, args&: useDeviceAddrTypes);
2040 args.useDevicePtrArgs.emplace(args&: useDevicePtrVars, args&: useDevicePtrTypes);
2041 printBlockArgRegion(p, op, region, args);
2042}
2043
2044template <typename ParsePrefixFn>
2045static ParseResult parseSplitIteratedList(
2046 OpAsmParser &parser,
2047 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &iteratedVars,
2048 SmallVectorImpl<Type> &iteratedTypes,
2049 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &plainVars,
2050 SmallVectorImpl<Type> &plainTypes, ParsePrefixFn &&parsePrefix) {
2051
2052 return parser.parseCommaSeparatedList([&]() -> ParseResult {
2053 if (failed(parsePrefix()))
2054 return failure();
2055
2056 OpAsmParser::UnresolvedOperand v;
2057 Type ty;
2058 if (parser.parseOperand(result&: v) || parser.parseColonType(result&: ty))
2059 return failure();
2060
2061 if (llvm::isa<mlir::omp::IteratedType>(Val: ty)) {
2062 iteratedVars.push_back(Elt: v);
2063 iteratedTypes.push_back(Elt: ty);
2064 } else {
2065 plainVars.push_back(Elt: v);
2066 plainTypes.push_back(Elt: ty);
2067 }
2068 return success();
2069 });
2070}
2071
2072template <typename PrintPrefixFn>
2073static void printSplitIteratedList(OpAsmPrinter &p, ValueRange iteratedVars,
2074 TypeRange iteratedTypes,
2075 ValueRange plainVars, TypeRange plainTypes,
2076 PrintPrefixFn &&printPrefixForPlain,
2077 PrintPrefixFn &&printPrefixForIterated) {
2078
2079 bool first = true;
2080 auto emit = [&](Value v, Type t, auto &&printPrefix) {
2081 if (!first)
2082 p << ", ";
2083 printPrefix(v, t);
2084 p << v << " : " << t;
2085 first = false;
2086 };
2087
2088 for (unsigned i = 0; i < iteratedVars.size(); ++i)
2089 emit(iteratedVars[i], iteratedTypes[i], printPrefixForIterated);
2090 for (unsigned i = 0; i < plainVars.size(); ++i)
2091 emit(plainVars[i], plainTypes[i], printPrefixForPlain);
2092}
2093
2094/// Verify the per-variable modifier list has one entry per reduction variable.
2095static LogicalResult verifyReductionModifiers(Operation *op,
2096 OperandRange reductionVars,
2097 ArrayAttr reductionMods) {
2098 if (reductionMods && reductionMods.size() != reductionVars.size())
2099 return op->emitOpError(
2100 message: "expected as many reduction modifiers as reduction variables");
2101 return success();
2102}
2103
2104/// Verifies Reduction Clause
2105static LogicalResult
2106verifyReductionVarList(Operation *op, std::optional<ArrayAttr> reductionSyms,
2107 OperandRange reductionVars,
2108 std::optional<ArrayRef<bool>> reductionByref,
2109 ArrayAttr reductionMods = nullptr) {
2110 if (failed(Result: verifyReductionModifiers(op, reductionVars, reductionMods)))
2111 return failure();
2112 if (!reductionVars.empty()) {
2113 if (!reductionSyms || reductionSyms->size() != reductionVars.size())
2114 return op->emitOpError()
2115 << "expected as many reduction symbol references "
2116 "as reduction variables";
2117 if (reductionByref && reductionByref->size() != reductionVars.size())
2118 return op->emitError() << "expected as many reduction variable by "
2119 "reference attributes as reduction variables";
2120 } else {
2121 if (reductionSyms)
2122 return op->emitOpError() << "unexpected reduction symbol references";
2123 return success();
2124 }
2125
2126 // TODO: The followings should be done in
2127 // SymbolUserOpInterface::verifySymbolUses.
2128 DenseSet<Value> accumulators;
2129 for (auto args : llvm::zip(t&: reductionVars, u&: *reductionSyms)) {
2130 Value accum = std::get<0>(t&: args);
2131
2132 if (!accumulators.insert(V: accum).second)
2133 return op->emitOpError() << "accumulator variable used more than once";
2134
2135 Type varType = accum.getType();
2136 auto symbolRef = llvm::cast<SymbolRefAttr>(Val: std::get<1>(t&: args));
2137 auto decl =
2138 SymbolTable::lookupNearestSymbolFrom<DeclareReductionOp>(from: op, symbol: symbolRef);
2139 if (!decl)
2140 return op->emitOpError() << "expected symbol reference " << symbolRef
2141 << " to point to a reduction declaration";
2142
2143 if (decl.getAccumulatorType() && decl.getAccumulatorType() != varType)
2144 return op->emitOpError()
2145 << "expected accumulator (" << varType
2146 << ") to be the same type as reduction declaration ("
2147 << decl.getAccumulatorType() << ")";
2148 }
2149
2150 return success();
2151}
2152
2153//===----------------------------------------------------------------------===//
2154// Parser, printer and verifier for Copyprivate
2155//===----------------------------------------------------------------------===//
2156
2157/// copyprivate-entry-list ::= copyprivate-entry
2158/// | copyprivate-entry-list `,` copyprivate-entry
2159/// copyprivate-entry ::= ssa-id `->` symbol-ref `:` type
2160static ParseResult parseCopyprivate(
2161 OpAsmParser &parser,
2162 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &copyprivateVars,
2163 SmallVectorImpl<Type> &copyprivateTypes, ArrayAttr &copyprivateSyms) {
2164 SmallVector<SymbolRefAttr> symsVec;
2165 if (failed(Result: parser.parseCommaSeparatedList(parseElementFn: [&]() {
2166 if (parser.parseOperand(result&: copyprivateVars.emplace_back()) ||
2167 parser.parseArrow() ||
2168 parser.parseAttribute(result&: symsVec.emplace_back()) ||
2169 parser.parseColonType(result&: copyprivateTypes.emplace_back()))
2170 return failure();
2171 return success();
2172 })))
2173 return failure();
2174 SmallVector<Attribute> syms(symsVec.begin(), symsVec.end());
2175 copyprivateSyms = ArrayAttr::get(context: parser.getContext(), value: syms);
2176 return success();
2177}
2178
2179/// Print Copyprivate clause
2180static void printCopyprivate(OpAsmPrinter &p, Operation *op,
2181 OperandRange copyprivateVars,
2182 TypeRange copyprivateTypes,
2183 std::optional<ArrayAttr> copyprivateSyms) {
2184 if (!copyprivateSyms.has_value())
2185 return;
2186 llvm::interleaveComma(
2187 c: llvm::zip(t&: copyprivateVars, u&: *copyprivateSyms, args&: copyprivateTypes), os&: p,
2188 each_fn: [&](const auto &args) {
2189 p << std::get<0>(args) << " -> " << std::get<1>(args) << " : "
2190 << std::get<2>(args);
2191 });
2192}
2193
2194/// Verifies CopyPrivate Clause
2195static LogicalResult
2196verifyCopyprivateVarList(Operation *op, OperandRange copyprivateVars,
2197 std::optional<ArrayAttr> copyprivateSyms) {
2198 size_t copyprivateSymsSize =
2199 copyprivateSyms.has_value() ? copyprivateSyms->size() : 0;
2200 if (copyprivateSymsSize != copyprivateVars.size())
2201 return op->emitOpError() << "inconsistent number of copyprivate vars (= "
2202 << copyprivateVars.size()
2203 << ") and functions (= " << copyprivateSymsSize
2204 << "), both must be equal";
2205 if (!copyprivateSyms.has_value())
2206 return success();
2207
2208 for (auto copyprivateVarAndSym :
2209 llvm::zip(t&: copyprivateVars, u&: *copyprivateSyms)) {
2210 auto symbolRef =
2211 llvm::cast<SymbolRefAttr>(Val: std::get<1>(t&: copyprivateVarAndSym));
2212 std::optional<std::variant<mlir::func::FuncOp, mlir::LLVM::LLVMFuncOp>>
2213 funcOp;
2214 if (mlir::func::FuncOp mlirFuncOp =
2215 SymbolTable::lookupNearestSymbolFrom<mlir::func::FuncOp>(from: op,
2216 symbol: symbolRef))
2217 funcOp = mlirFuncOp;
2218 else if (mlir::LLVM::LLVMFuncOp llvmFuncOp =
2219 SymbolTable::lookupNearestSymbolFrom<mlir::LLVM::LLVMFuncOp>(
2220 from: op, symbol: symbolRef))
2221 funcOp = llvmFuncOp;
2222
2223 auto getNumArguments = [&] {
2224 return std::visit(visitor: [](auto &f) { return f.getNumArguments(); }, variants&: *funcOp);
2225 };
2226
2227 auto getArgumentType = [&](unsigned i) {
2228 return std::visit(visitor: [i](auto &f) { return f.getArgumentTypes()[i]; },
2229 variants&: *funcOp);
2230 };
2231
2232 if (!funcOp)
2233 return op->emitOpError() << "expected symbol reference " << symbolRef
2234 << " to point to a copy function";
2235
2236 if (getNumArguments() != 2)
2237 return op->emitOpError()
2238 << "expected copy function " << symbolRef << " to have 2 operands";
2239
2240 Type argTy = getArgumentType(0);
2241 if (argTy != getArgumentType(1))
2242 return op->emitOpError() << "expected copy function " << symbolRef
2243 << " arguments to have the same type";
2244
2245 Type varType = std::get<0>(t&: copyprivateVarAndSym).getType();
2246 if (argTy != varType)
2247 return op->emitOpError()
2248 << "expected copy function arguments' type (" << argTy
2249 << ") to be the same as copyprivate variable's type (" << varType
2250 << ")";
2251 }
2252
2253 return success();
2254}
2255
2256//===----------------------------------------------------------------------===//
2257// Parser, printer and verifier for DependVarList
2258//===----------------------------------------------------------------------===//
2259
2260/// depend-entry-list ::= depend-entry
2261/// | depend-entry-list `,` depend-entry
2262/// depend-entry ::= depend-kind `->` ssa-id `:` type
2263/// | depend-kind `->` ssa-id `:` iterated-type
2264static ParseResult parseDependVarList(
2265 OpAsmParser &parser,
2266 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &dependVars,
2267 SmallVectorImpl<Type> &dependTypes, ArrayAttr &dependKinds,
2268 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &iteratedVars,
2269 SmallVectorImpl<Type> &iteratedTypes, ArrayAttr &iteratedKinds) {
2270 SmallVector<ClauseTaskDependAttr> kindsVec;
2271 SmallVector<ClauseTaskDependAttr> iterKindsVec;
2272 if (failed(Result: parser.parseCommaSeparatedList(parseElementFn: [&]() {
2273 StringRef keyword;
2274 OpAsmParser::UnresolvedOperand operand;
2275 Type ty;
2276 if (parser.parseKeyword(keyword: &keyword) || parser.parseArrow() ||
2277 parser.parseOperand(result&: operand) || parser.parseColonType(result&: ty))
2278 return failure();
2279 std::optional<ClauseTaskDepend> keywordDepend =
2280 symbolizeClauseTaskDepend(str: keyword);
2281 if (!keywordDepend)
2282 return failure();
2283 auto kindAttr =
2284 ClauseTaskDependAttr::get(context: parser.getContext(), value: *keywordDepend);
2285 if (llvm::isa<mlir::omp::IteratedType>(Val: ty)) {
2286 iteratedVars.push_back(Elt: operand);
2287 iteratedTypes.push_back(Elt: ty);
2288 iterKindsVec.push_back(Elt: kindAttr);
2289 } else {
2290 dependVars.push_back(Elt: operand);
2291 dependTypes.push_back(Elt: ty);
2292 kindsVec.push_back(Elt: kindAttr);
2293 }
2294 return success();
2295 })))
2296 return failure();
2297 SmallVector<Attribute> kinds(kindsVec.begin(), kindsVec.end());
2298 dependKinds = ArrayAttr::get(context: parser.getContext(), value: kinds);
2299 SmallVector<Attribute> iterKinds(iterKindsVec.begin(), iterKindsVec.end());
2300 iteratedKinds = ArrayAttr::get(context: parser.getContext(), value: iterKinds);
2301 return success();
2302}
2303
2304/// Print Depend clause
2305static void printDependVarList(OpAsmPrinter &p, Operation *op,
2306 OperandRange dependVars, TypeRange dependTypes,
2307 std::optional<ArrayAttr> dependKinds,
2308 OperandRange iteratedVars,
2309 TypeRange iteratedTypes,
2310 std::optional<ArrayAttr> iteratedKinds) {
2311 bool first = true;
2312 auto printEntries = [&](OperandRange vars, TypeRange types,
2313 std::optional<ArrayAttr> kinds) {
2314 for (unsigned i = 0, e = vars.size(); i < e; ++i) {
2315 if (!first)
2316 p << ", ";
2317 p << stringifyClauseTaskDepend(
2318 val: llvm::cast<mlir::omp::ClauseTaskDependAttr>(Val: (*kinds)[i])
2319 .getValue())
2320 << " -> " << vars[i] << " : " << types[i];
2321 first = false;
2322 }
2323 };
2324 printEntries(dependVars, dependTypes, dependKinds);
2325 printEntries(iteratedVars, iteratedTypes, iteratedKinds);
2326}
2327
2328/// Verifies Depend clause
2329static LogicalResult verifyDependVarList(Operation *op,
2330 std::optional<ArrayAttr> dependKinds,
2331 OperandRange dependVars,
2332 std::optional<ArrayAttr> iteratedKinds,
2333 OperandRange iteratedVars) {
2334 if (!dependVars.empty()) {
2335 if (!dependKinds || dependKinds->size() != dependVars.size())
2336 return op->emitOpError() << "expected as many depend values"
2337 " as depend variables";
2338 } else {
2339 if (dependKinds && !dependKinds->empty())
2340 return op->emitOpError() << "unexpected depend values";
2341 }
2342
2343 if (!iteratedVars.empty()) {
2344 if (!iteratedKinds || iteratedKinds->size() != iteratedVars.size())
2345 return op->emitOpError() << "expected as many depend iterated values"
2346 " as depend iterated variables";
2347 } else {
2348 if (iteratedKinds && !iteratedKinds->empty())
2349 return op->emitOpError() << "unexpected depend iterated values";
2350 }
2351
2352 return success();
2353}
2354
2355//===----------------------------------------------------------------------===//
2356// Parser, printer and verifier for Synchronization Hint (2.17.12)
2357//===----------------------------------------------------------------------===//
2358
2359/// Parses a Synchronization Hint clause. The value of hint is an integer
2360/// which is a combination of different hints from `omp_sync_hint_t`.
2361///
2362/// hint-clause = `hint` `(` hint-value `)`
2363static ParseResult parseSynchronizationHint(OpAsmParser &parser,
2364 IntegerAttr &hintAttr) {
2365 StringRef hintKeyword;
2366 int64_t hint = 0;
2367 if (succeeded(Result: parser.parseOptionalKeyword(keyword: "none"))) {
2368 hintAttr = IntegerAttr::get(type: parser.getBuilder().getI64Type(), value: 0);
2369 return success();
2370 }
2371 auto parseKeyword = [&]() -> ParseResult {
2372 if (failed(Result: parser.parseKeyword(keyword: &hintKeyword)))
2373 return failure();
2374 if (hintKeyword == "uncontended")
2375 hint |= 1;
2376 else if (hintKeyword == "contended")
2377 hint |= 2;
2378 else if (hintKeyword == "nonspeculative")
2379 hint |= 4;
2380 else if (hintKeyword == "speculative")
2381 hint |= 8;
2382 else
2383 return parser.emitError(loc: parser.getCurrentLocation())
2384 << hintKeyword << " is not a valid hint";
2385 return success();
2386 };
2387 if (parser.parseCommaSeparatedList(parseElementFn: parseKeyword))
2388 return failure();
2389 hintAttr = IntegerAttr::get(type: parser.getBuilder().getI64Type(), value: hint);
2390 return success();
2391}
2392
2393/// Prints a Synchronization Hint clause
2394static void printSynchronizationHint(OpAsmPrinter &p, Operation *op,
2395 IntegerAttr hintAttr) {
2396 int64_t hint = hintAttr.getInt();
2397
2398 if (hint == 0) {
2399 p << "none";
2400 return;
2401 }
2402
2403 // Helper function to get n-th bit from the right end of `value`
2404 auto bitn = [](int value, int n) -> bool { return value & (1 << n); };
2405
2406 bool uncontended = bitn(hint, 0);
2407 bool contended = bitn(hint, 1);
2408 bool nonspeculative = bitn(hint, 2);
2409 bool speculative = bitn(hint, 3);
2410
2411 SmallVector<StringRef> hints;
2412 if (uncontended)
2413 hints.push_back(Elt: "uncontended");
2414 if (contended)
2415 hints.push_back(Elt: "contended");
2416 if (nonspeculative)
2417 hints.push_back(Elt: "nonspeculative");
2418 if (speculative)
2419 hints.push_back(Elt: "speculative");
2420
2421 llvm::interleaveComma(c: hints, os&: p);
2422}
2423
2424/// Verifies a synchronization hint clause
2425static LogicalResult verifySynchronizationHint(Operation *op, uint64_t hint) {
2426
2427 // Helper function to get n-th bit from the right end of `value`
2428 auto bitn = [](int value, int n) -> bool { return value & (1 << n); };
2429
2430 bool uncontended = bitn(hint, 0);
2431 bool contended = bitn(hint, 1);
2432 bool nonspeculative = bitn(hint, 2);
2433 bool speculative = bitn(hint, 3);
2434
2435 if (uncontended && contended)
2436 return op->emitOpError() << "the hints omp_sync_hint_uncontended and "
2437 "omp_sync_hint_contended cannot be combined";
2438 if (nonspeculative && speculative)
2439 return op->emitOpError() << "the hints omp_sync_hint_nonspeculative and "
2440 "omp_sync_hint_speculative cannot be combined.";
2441 return success();
2442}
2443
2444//===----------------------------------------------------------------------===//
2445// Parser, printer and verifier for Target
2446//===----------------------------------------------------------------------===//
2447
2448// Helper function to get bitwise AND of `value` and 'flag' then return it as a
2449// boolean
2450static bool mapTypeToBool(ClauseMapFlags value, ClauseMapFlags flag) {
2451 return (value & flag) == flag;
2452}
2453
2454/// Parses a map_entries map type from a string format back into its numeric
2455/// value.
2456///
2457/// map-clause = `map_clauses ( ( `(` `always, `? `implicit, `? `ompx_hold, `?
2458/// `close, `? `present, `? ( `to` | `from` | `delete` `)` )+ `)` )
2459static ParseResult parseMapClause(OpAsmParser &parser,
2460 ClauseMapFlagsAttr &mapType) {
2461 ClauseMapFlags mapTypeBits = ClauseMapFlags::none;
2462 // This simply verifies the correct keyword is read in, the
2463 // keyword itself is stored inside of the operation
2464 auto parseTypeAndMod = [&]() -> ParseResult {
2465 StringRef mapTypeMod;
2466 if (parser.parseKeyword(keyword: &mapTypeMod))
2467 return failure();
2468
2469 if (mapTypeMod == "always")
2470 mapTypeBits |= ClauseMapFlags::always;
2471
2472 if (mapTypeMod == "implicit")
2473 mapTypeBits |= ClauseMapFlags::implicit;
2474
2475 if (mapTypeMod == "ompx_hold")
2476 mapTypeBits |= ClauseMapFlags::ompx_hold;
2477
2478 if (mapTypeMod == "close")
2479 mapTypeBits |= ClauseMapFlags::close;
2480
2481 if (mapTypeMod == "present")
2482 mapTypeBits |= ClauseMapFlags::present;
2483
2484 if (mapTypeMod == "to")
2485 mapTypeBits |= ClauseMapFlags::to;
2486
2487 if (mapTypeMod == "from")
2488 mapTypeBits |= ClauseMapFlags::from;
2489
2490 if (mapTypeMod == "tofrom")
2491 mapTypeBits |= ClauseMapFlags::to | ClauseMapFlags::from;
2492
2493 if (mapTypeMod == "delete")
2494 mapTypeBits |= ClauseMapFlags::del;
2495
2496 if (mapTypeMod == "storage")
2497 mapTypeBits |= ClauseMapFlags::storage;
2498
2499 if (mapTypeMod == "return_param")
2500 mapTypeBits |= ClauseMapFlags::return_param;
2501
2502 if (mapTypeMod == "private")
2503 mapTypeBits |= ClauseMapFlags::priv;
2504
2505 if (mapTypeMod == "literal")
2506 mapTypeBits |= ClauseMapFlags::literal;
2507
2508 if (mapTypeMod == "attach")
2509 mapTypeBits |= ClauseMapFlags::attach;
2510
2511 if (mapTypeMod == "attach_always")
2512 mapTypeBits |= ClauseMapFlags::attach_always;
2513
2514 if (mapTypeMod == "attach_never")
2515 mapTypeBits |= ClauseMapFlags::attach_never;
2516
2517 if (mapTypeMod == "attach_auto")
2518 mapTypeBits |= ClauseMapFlags::attach_auto;
2519
2520 if (mapTypeMod == "ref_ptr")
2521 mapTypeBits |= ClauseMapFlags::ref_ptr;
2522
2523 if (mapTypeMod == "ref_ptee")
2524 mapTypeBits |= ClauseMapFlags::ref_ptee;
2525
2526 if (mapTypeMod == "is_device_ptr")
2527 mapTypeBits |= ClauseMapFlags::is_device_ptr;
2528
2529 if (mapTypeMod == "target_param")
2530 mapTypeBits |= ClauseMapFlags::target_param;
2531
2532 return success();
2533 };
2534
2535 if (parser.parseCommaSeparatedList(parseElementFn: parseTypeAndMod))
2536 return failure();
2537
2538 mapType =
2539 parser.getBuilder().getAttr<mlir::omp::ClauseMapFlagsAttr>(args&: mapTypeBits);
2540
2541 return success();
2542}
2543
2544/// Prints a map_entries map type from its numeric value out into its string
2545/// format.
2546static void printMapClause(OpAsmPrinter &p, Operation *op,
2547 ClauseMapFlagsAttr mapType) {
2548 llvm::SmallVector<std::string, 4> mapTypeStrs;
2549 ClauseMapFlags mapFlags = mapType.getValue();
2550
2551 // handling of always, close, present placed at the beginning of the string
2552 // to aid readability
2553 if (mapTypeToBool(value: mapFlags, flag: ClauseMapFlags::always))
2554 mapTypeStrs.push_back(Elt: "always");
2555 if (mapTypeToBool(value: mapFlags, flag: ClauseMapFlags::implicit))
2556 mapTypeStrs.push_back(Elt: "implicit");
2557 if (mapTypeToBool(value: mapFlags, flag: ClauseMapFlags::ompx_hold))
2558 mapTypeStrs.push_back(Elt: "ompx_hold");
2559 if (mapTypeToBool(value: mapFlags, flag: ClauseMapFlags::close))
2560 mapTypeStrs.push_back(Elt: "close");
2561 if (mapTypeToBool(value: mapFlags, flag: ClauseMapFlags::present))
2562 mapTypeStrs.push_back(Elt: "present");
2563 if (mapTypeToBool(value: mapFlags, flag: ClauseMapFlags::target_param))
2564 mapTypeStrs.push_back(Elt: "target_param");
2565
2566 // special handling of to/from/tofrom/delete and release/alloc, release +
2567 // alloc are the abscense of one of the other flags, whereas tofrom requires
2568 // both the to and from flag to be set.
2569 bool to = mapTypeToBool(value: mapFlags, flag: ClauseMapFlags::to);
2570 bool from = mapTypeToBool(value: mapFlags, flag: ClauseMapFlags::from);
2571
2572 if (to && from)
2573 mapTypeStrs.push_back(Elt: "tofrom");
2574 else if (from)
2575 mapTypeStrs.push_back(Elt: "from");
2576 else if (to)
2577 mapTypeStrs.push_back(Elt: "to");
2578
2579 if (mapTypeToBool(value: mapFlags, flag: ClauseMapFlags::del))
2580 mapTypeStrs.push_back(Elt: "delete");
2581 if (mapTypeToBool(value: mapFlags, flag: ClauseMapFlags::return_param))
2582 mapTypeStrs.push_back(Elt: "return_param");
2583 if (mapTypeToBool(value: mapFlags, flag: ClauseMapFlags::storage))
2584 mapTypeStrs.push_back(Elt: "storage");
2585 if (mapTypeToBool(value: mapFlags, flag: ClauseMapFlags::priv))
2586 mapTypeStrs.push_back(Elt: "private");
2587 if (mapTypeToBool(value: mapFlags, flag: ClauseMapFlags::literal))
2588 mapTypeStrs.push_back(Elt: "literal");
2589 if (mapTypeToBool(value: mapFlags, flag: ClauseMapFlags::attach))
2590 mapTypeStrs.push_back(Elt: "attach");
2591 if (mapTypeToBool(value: mapFlags, flag: ClauseMapFlags::attach_always))
2592 mapTypeStrs.push_back(Elt: "attach_always");
2593 if (mapTypeToBool(value: mapFlags, flag: ClauseMapFlags::attach_never))
2594 mapTypeStrs.push_back(Elt: "attach_never");
2595 if (mapTypeToBool(value: mapFlags, flag: ClauseMapFlags::attach_auto))
2596 mapTypeStrs.push_back(Elt: "attach_auto");
2597 if (mapTypeToBool(value: mapFlags, flag: ClauseMapFlags::ref_ptr))
2598 mapTypeStrs.push_back(Elt: "ref_ptr");
2599 if (mapTypeToBool(value: mapFlags, flag: ClauseMapFlags::ref_ptee))
2600 mapTypeStrs.push_back(Elt: "ref_ptee");
2601 if (mapTypeToBool(value: mapFlags, flag: ClauseMapFlags::is_device_ptr))
2602 mapTypeStrs.push_back(Elt: "is_device_ptr");
2603 if (mapFlags == ClauseMapFlags::none)
2604 mapTypeStrs.push_back(Elt: "none");
2605
2606 for (unsigned int i = 0; i < mapTypeStrs.size(); ++i) {
2607 p << mapTypeStrs[i];
2608 if (i + 1 < mapTypeStrs.size()) {
2609 p << ", ";
2610 }
2611 }
2612}
2613
2614static ParseResult parseMembersIndex(OpAsmParser &parser,
2615 ArrayAttr &membersIdx) {
2616 SmallVector<Attribute> values, memberIdxs;
2617
2618 auto parseIndices = [&]() -> ParseResult {
2619 int64_t value;
2620 if (parser.parseInteger(result&: value))
2621 return failure();
2622 values.push_back(Elt: IntegerAttr::get(type: parser.getBuilder().getIntegerType(width: 64),
2623 value: APInt(64, value, /*isSigned=*/false)));
2624 return success();
2625 };
2626
2627 do {
2628 if (failed(Result: parser.parseLSquare()))
2629 return failure();
2630
2631 if (parser.parseCommaSeparatedList(parseElementFn: parseIndices))
2632 return failure();
2633
2634 if (failed(Result: parser.parseRSquare()))
2635 return failure();
2636
2637 memberIdxs.push_back(Elt: ArrayAttr::get(context: parser.getContext(), value: values));
2638 values.clear();
2639 } while (succeeded(Result: parser.parseOptionalComma()));
2640
2641 if (!memberIdxs.empty())
2642 membersIdx = ArrayAttr::get(context: parser.getContext(), value: memberIdxs);
2643
2644 return success();
2645}
2646
2647static void printMembersIndex(OpAsmPrinter &p, MapInfoOp op,
2648 ArrayAttr membersIdx) {
2649 if (!membersIdx)
2650 return;
2651
2652 llvm::interleaveComma(c: membersIdx, os&: p, each_fn: [&p](Attribute v) {
2653 p << "[";
2654 auto memberIdx = cast<ArrayAttr>(Val&: v);
2655 llvm::interleaveComma(c: memberIdx.getValue(), os&: p, each_fn: [&p](Attribute v2) {
2656 p << cast<IntegerAttr>(Val&: v2).getInt();
2657 });
2658 p << "]";
2659 });
2660}
2661
2662static void printCaptureType(OpAsmPrinter &p, Operation *op,
2663 VariableCaptureKindAttr mapCaptureType) {
2664 std::string typeCapStr;
2665 llvm::raw_string_ostream typeCap(typeCapStr);
2666 if (mapCaptureType.getValue() == mlir::omp::VariableCaptureKind::ByRef)
2667 typeCap << "ByRef";
2668 if (mapCaptureType.getValue() == mlir::omp::VariableCaptureKind::ByCopy)
2669 typeCap << "ByCopy";
2670 if (mapCaptureType.getValue() == mlir::omp::VariableCaptureKind::VLAType)
2671 typeCap << "VLAType";
2672 if (mapCaptureType.getValue() == mlir::omp::VariableCaptureKind::This)
2673 typeCap << "This";
2674 p << typeCapStr;
2675}
2676
2677static ParseResult parseCaptureType(OpAsmParser &parser,
2678 VariableCaptureKindAttr &mapCaptureType) {
2679 StringRef mapCaptureKey;
2680 if (parser.parseKeyword(keyword: &mapCaptureKey))
2681 return failure();
2682
2683 if (mapCaptureKey == "This")
2684 mapCaptureType = mlir::omp::VariableCaptureKindAttr::get(
2685 context: parser.getContext(), value: mlir::omp::VariableCaptureKind::This);
2686 if (mapCaptureKey == "ByRef")
2687 mapCaptureType = mlir::omp::VariableCaptureKindAttr::get(
2688 context: parser.getContext(), value: mlir::omp::VariableCaptureKind::ByRef);
2689 if (mapCaptureKey == "ByCopy")
2690 mapCaptureType = mlir::omp::VariableCaptureKindAttr::get(
2691 context: parser.getContext(), value: mlir::omp::VariableCaptureKind::ByCopy);
2692 if (mapCaptureKey == "VLAType")
2693 mapCaptureType = mlir::omp::VariableCaptureKindAttr::get(
2694 context: parser.getContext(), value: mlir::omp::VariableCaptureKind::VLAType);
2695
2696 return success();
2697}
2698
2699static LogicalResult verifyMapInfoForMapClause(
2700 Operation *op, mlir::omp::MapInfoOp mapInfoOp,
2701 llvm::DenseSet<mlir::TypedValue<mlir::omp::PointerLikeType>> &updateToVars,
2702 llvm::DenseSet<mlir::TypedValue<mlir::omp::PointerLikeType>>
2703 &updateFromVars) {
2704 mlir::omp::ClauseMapFlags mapTypeBits = mapInfoOp.getMapType();
2705
2706 bool to = mapTypeToBool(value: mapTypeBits, flag: ClauseMapFlags::to);
2707 bool from = mapTypeToBool(value: mapTypeBits, flag: ClauseMapFlags::from);
2708 bool del = mapTypeToBool(value: mapTypeBits, flag: ClauseMapFlags::del);
2709
2710 bool always = mapTypeToBool(value: mapTypeBits, flag: ClauseMapFlags::always);
2711 bool close = mapTypeToBool(value: mapTypeBits, flag: ClauseMapFlags::close);
2712 bool implicit = mapTypeToBool(value: mapTypeBits, flag: ClauseMapFlags::implicit);
2713 bool attach = mapTypeToBool(value: mapTypeBits, flag: ClauseMapFlags::attach);
2714
2715 if ((isa<TargetDataOp>(Val: op) || isa<TargetOp>(Val: op)) && del)
2716 return emitError(loc: op->getLoc(),
2717 message: "to, from, tofrom and alloc map types are permitted");
2718
2719 if (isa<TargetEnterDataOp>(Val: op) && (from || del))
2720 return emitError(loc: op->getLoc(), message: "to and alloc map types are permitted");
2721
2722 if (isa<TargetExitDataOp>(Val: op) && to)
2723 return emitError(loc: op->getLoc(),
2724 message: "from, release and delete map types are permitted");
2725
2726 if (isa<TargetUpdateOp>(Val: op)) {
2727 if (del) {
2728 return emitError(loc: op->getLoc(),
2729 message: "at least one of to or from map types must be "
2730 "specified, other map types are not permitted");
2731 }
2732
2733 if (!to && !from && !attach) {
2734 return emitError(loc: op->getLoc(),
2735 message: "at least one of to or from or attach map types must be "
2736 "specified, other map types are not permitted");
2737 }
2738
2739 auto updateVar = mapInfoOp.getVarPtr();
2740
2741 if ((to && from) || (to && updateFromVars.contains(V: updateVar)) ||
2742 (from && updateToVars.contains(V: updateVar))) {
2743 return emitError(
2744 loc: op->getLoc(),
2745 message: "either to or from map types can be specified, not both");
2746 }
2747
2748 if (always || close || implicit) {
2749 return emitError(
2750 loc: op->getLoc(),
2751 message: "present, mapper and iterator map type modifiers are permitted");
2752 }
2753
2754 // It's possible we have an attach map, in which case if there is no to
2755 // or from tied to it, we skip insertion.
2756 if (to || from) {
2757 to ? updateToVars.insert(V: updateVar) : updateFromVars.insert(V: updateVar);
2758 }
2759 }
2760
2761 if ((mapInfoOp.getVarPtrPtr() && !mapInfoOp.getVarPtrPtrType()) ||
2762 (!mapInfoOp.getVarPtrPtr() && mapInfoOp.getVarPtrPtrType())) {
2763 return emitError(loc: op->getLoc(),
2764 message: "if varPtrPtr or varPtrPtrType is specified, then both "
2765 "must be present");
2766 }
2767
2768 return success();
2769}
2770
2771static LogicalResult verifyMapClause(Operation *op, OperandRange mapVars,
2772 OperandRange mapIterated) {
2773 llvm::DenseSet<mlir::TypedValue<mlir::omp::PointerLikeType>> updateToVars;
2774 llvm::DenseSet<mlir::TypedValue<mlir::omp::PointerLikeType>> updateFromVars;
2775
2776 for (auto mapOp : mapVars) {
2777 if (!mapOp.getDefiningOp())
2778 return emitError(loc: op->getLoc(), message: "missing map operation");
2779
2780 if (auto mapInfoOp = mapOp.getDefiningOp<mlir::omp::MapInfoOp>()) {
2781 if (failed(Result: verifyMapInfoForMapClause(op, mapInfoOp, updateToVars,
2782 updateFromVars)))
2783 return failure();
2784 } else if (!isa<DeclareMapperInfoOp>(Val: op)) {
2785 return emitError(loc: op->getLoc(),
2786 message: "map argument is not a map entry operation");
2787 }
2788 }
2789
2790 // Verify iterated map entries.
2791 for (auto iterVal : mapIterated) {
2792 auto iterOp = iterVal.getDefiningOp<mlir::omp::IteratorOp>();
2793 if (!iterOp)
2794 return op->emitOpError() << "'map_iterated' arguments must be defined by "
2795 "'omp.iterator' ops";
2796
2797 // Check that the iterator body yields a value defined by omp.map.info.
2798 mlir::omp::YieldOp yieldOp = iterOp.getYieldOp();
2799 auto yieldedMapInfo =
2800 yieldOp && yieldOp.getNumOperands() == 1
2801 ? yieldOp.getResults()[0].getDefiningOp<mlir::omp::MapInfoOp>()
2802 : nullptr;
2803 if (!yieldedMapInfo)
2804 return op->emitOpError() << "'map_iterated' iterator body must yield "
2805 "a value defined by 'omp.map.info'";
2806
2807 if (failed(Result: verifyMapInfoForMapClause(op, mapInfoOp: yieldedMapInfo, updateToVars,
2808 updateFromVars)))
2809 return failure();
2810 }
2811
2812 return success();
2813}
2814
2815template <typename OpType>
2816static LogicalResult verifyPrivateVarList(OpType &op);
2817
2818static LogicalResult verifyPrivateVarsMapping(TargetOp targetOp) {
2819 std::optional<DenseI64ArrayAttr> privateMapIndices =
2820 targetOp.getPrivateMapsAttr();
2821
2822 // None of the private operands are mapped.
2823 if (!privateMapIndices.has_value() || !privateMapIndices.value())
2824 return success();
2825
2826 OperandRange privateVars = targetOp.getPrivateVars();
2827
2828 if (privateMapIndices.value().size() !=
2829 static_cast<int64_t>(privateVars.size()))
2830 return emitError(loc: targetOp.getLoc(), message: "sizes of `private` operand range and "
2831 "`private_maps` attribute mismatch");
2832
2833 return success();
2834}
2835
2836//===----------------------------------------------------------------------===//
2837// MapInfoOp
2838//===----------------------------------------------------------------------===//
2839
2840static LogicalResult verifyMapInfoDefinedArgs(Operation *op,
2841 StringRef clauseName,
2842 OperandRange vars) {
2843 for (Value var : vars)
2844 if (!llvm::isa_and_present<MapInfoOp>(Val: var.getDefiningOp()))
2845 return op->emitOpError()
2846 << "'" << clauseName
2847 << "' arguments must be defined by 'omp.map.info' ops";
2848 return success();
2849}
2850
2851LogicalResult MapInfoOp::verify() {
2852 if (getMapperId() &&
2853 !SymbolTable::lookupNearestSymbolFrom<omp::DeclareMapperOp>(
2854 from: *this, symbol: getMapperIdAttr())) {
2855 return emitError(message: "invalid mapper id");
2856 }
2857
2858 if (failed(Result: verifyMapInfoDefinedArgs(op: *this, clauseName: "members", vars: getMembers())))
2859 return failure();
2860
2861 return success();
2862}
2863
2864//===----------------------------------------------------------------------===//
2865// TargetDataOp
2866//===----------------------------------------------------------------------===//
2867
2868void TargetDataOp::build(OpBuilder &builder, OperationState &state,
2869 const TargetDataOperands &clauses) {
2870 TargetDataOp::build(odsBuilder&: builder, odsState&: state, device: clauses.device, if_expr: clauses.ifExpr,
2871 map_vars: clauses.mapVars, map_iterated: clauses.mapIterated,
2872 use_device_addr_vars: clauses.useDeviceAddrVars, use_device_ptr_vars: clauses.useDevicePtrVars);
2873}
2874
2875LogicalResult TargetDataOp::verify() {
2876 if (getMapVars().empty() && getMapIterated().empty() &&
2877 getUseDevicePtrVars().empty() && getUseDeviceAddrVars().empty()) {
2878 return ::emitError(loc: this->getLoc(),
2879 message: "At least one of map, use_device_ptr_vars, or "
2880 "use_device_addr_vars operand must be present");
2881 }
2882
2883 if (failed(Result: verifyMapInfoDefinedArgs(op: *this, clauseName: "use_device_ptr",
2884 vars: getUseDevicePtrVars())))
2885 return failure();
2886
2887 if (failed(Result: verifyMapInfoDefinedArgs(op: *this, clauseName: "use_device_addr",
2888 vars: getUseDeviceAddrVars())))
2889 return failure();
2890
2891 return verifyMapClause(op: *this, mapVars: getMapVars(), mapIterated: getMapIterated());
2892}
2893
2894//===----------------------------------------------------------------------===//
2895// TargetEnterDataOp
2896//===----------------------------------------------------------------------===//
2897
2898void TargetEnterDataOp::build(
2899 OpBuilder &builder, OperationState &state,
2900 const TargetEnterExitUpdateDataOperands &clauses) {
2901 MLIRContext *ctx = builder.getContext();
2902 TargetEnterDataOp::build(
2903 odsBuilder&: builder, odsState&: state, depend_kinds: makeArrayAttr(context: ctx, attrs: clauses.dependKinds),
2904 depend_vars: clauses.dependVars, depend_iterated_kinds: makeArrayAttr(context: ctx, attrs: clauses.dependIteratedKinds),
2905 depend_iterated: clauses.dependIterated, device: clauses.device, if_expr: clauses.ifExpr, map_vars: clauses.mapVars,
2906 map_iterated: clauses.mapIterated, nowait: clauses.nowait);
2907}
2908
2909LogicalResult TargetEnterDataOp::verify() {
2910 LogicalResult verifyDependVars =
2911 verifyDependVarList(op: *this, dependKinds: getDependKinds(), dependVars: getDependVars(),
2912 iteratedKinds: getDependIteratedKinds(), iteratedVars: getDependIterated());
2913 return failed(Result: verifyDependVars)
2914 ? verifyDependVars
2915 : verifyMapClause(op: *this, mapVars: getMapVars(), mapIterated: getMapIterated());
2916}
2917
2918//===----------------------------------------------------------------------===//
2919// TargetExitDataOp
2920//===----------------------------------------------------------------------===//
2921
2922void TargetExitDataOp::build(OpBuilder &builder, OperationState &state,
2923 const TargetEnterExitUpdateDataOperands &clauses) {
2924 MLIRContext *ctx = builder.getContext();
2925 TargetExitDataOp::build(
2926 odsBuilder&: builder, odsState&: state, depend_kinds: makeArrayAttr(context: ctx, attrs: clauses.dependKinds),
2927 depend_vars: clauses.dependVars, depend_iterated_kinds: makeArrayAttr(context: ctx, attrs: clauses.dependIteratedKinds),
2928 depend_iterated: clauses.dependIterated, device: clauses.device, if_expr: clauses.ifExpr, map_vars: clauses.mapVars,
2929 map_iterated: clauses.mapIterated, nowait: clauses.nowait);
2930}
2931
2932LogicalResult TargetExitDataOp::verify() {
2933 LogicalResult verifyDependVars =
2934 verifyDependVarList(op: *this, dependKinds: getDependKinds(), dependVars: getDependVars(),
2935 iteratedKinds: getDependIteratedKinds(), iteratedVars: getDependIterated());
2936 return failed(Result: verifyDependVars)
2937 ? verifyDependVars
2938 : verifyMapClause(op: *this, mapVars: getMapVars(), mapIterated: getMapIterated());
2939}
2940
2941//===----------------------------------------------------------------------===//
2942// TargetUpdateOp
2943//===----------------------------------------------------------------------===//
2944
2945void TargetUpdateOp::build(OpBuilder &builder, OperationState &state,
2946 const TargetEnterExitUpdateDataOperands &clauses) {
2947 MLIRContext *ctx = builder.getContext();
2948 TargetUpdateOp::build(odsBuilder&: builder, odsState&: state, depend_kinds: makeArrayAttr(context: ctx, attrs: clauses.dependKinds),
2949 depend_vars: clauses.dependVars,
2950 depend_iterated_kinds: makeArrayAttr(context: ctx, attrs: clauses.dependIteratedKinds),
2951 depend_iterated: clauses.dependIterated, device: clauses.device, if_expr: clauses.ifExpr,
2952 map_vars: clauses.mapVars, map_iterated: clauses.mapIterated, nowait: clauses.nowait);
2953}
2954
2955LogicalResult TargetUpdateOp::verify() {
2956 LogicalResult verifyDependVars =
2957 verifyDependVarList(op: *this, dependKinds: getDependKinds(), dependVars: getDependVars(),
2958 iteratedKinds: getDependIteratedKinds(), iteratedVars: getDependIterated());
2959 return failed(Result: verifyDependVars)
2960 ? verifyDependVars
2961 : verifyMapClause(op: *this, mapVars: getMapVars(), mapIterated: getMapIterated());
2962}
2963
2964//===----------------------------------------------------------------------===//
2965// TargetOp
2966//===----------------------------------------------------------------------===//
2967
2968void TargetOp::build(OpBuilder &builder, OperationState &state,
2969 const TargetExtOperands &clauses) {
2970 MLIRContext *ctx = builder.getContext();
2971 TargetOp::build(
2972 odsBuilder&: builder, odsState&: state, allocate_vars: clauses.allocateVars, allocator_vars: clauses.allocatorVars,
2973 allocate_alignments: makeDenseI64ArrayAttr(ctx, intArray: clauses.allocateAlignments),
2974 allocate_private_indices: makeDenseI64ArrayAttr(ctx, intArray: clauses.allocatePrivateIndices),
2975 depend_kinds: makeArrayAttr(context: ctx, attrs: clauses.dependKinds), depend_vars: clauses.dependVars,
2976 depend_iterated_kinds: makeArrayAttr(context: ctx, attrs: clauses.dependIteratedKinds), depend_iterated: clauses.dependIterated,
2977 device: clauses.device, dyn_groupprivate_access_group: clauses.dynGroupprivateAccessGroup,
2978 dyn_groupprivate_fallback: clauses.dynGroupprivateFallback, dyn_groupprivate_size: clauses.dynGroupprivateSize,
2979 has_device_addr_vars: clauses.hasDeviceAddrVars, host_eval_vars: clauses.hostEvalVars, if_expr: clauses.ifExpr,
2980 in_reduction_vars: clauses.inReductionVars,
2981 in_reduction_byref: makeDenseBoolArrayAttr(ctx, boolArray: clauses.inReductionByref),
2982 in_reduction_syms: makeArrayAttr(context: ctx, attrs: clauses.inReductionSyms), is_device_ptr_vars: clauses.isDevicePtrVars,
2983 map_vars: clauses.mapVars, map_iterated: clauses.mapIterated, nowait: clauses.nowait, private_vars: clauses.privateVars,
2984 private_syms: makeArrayAttr(context: ctx, attrs: clauses.privateSyms), private_needs_barrier: clauses.privateNeedsBarrier,
2985 thread_limit_vars: clauses.threadLimitVars, /*private_maps=*/nullptr, kernel_type: clauses.kernelType);
2986}
2987
2988bool TargetOp::hasHostEvalTripCount() {
2989 TargetExecMode mode = getKernelType();
2990 if (mode == TargetExecMode::spmd || mode == TargetExecMode::spmd_no_loop)
2991 return true;
2992
2993 if (mode == TargetExecMode::bare)
2994 return false;
2995
2996 // If it represents a `target teams distribute` construct, also evaluate the
2997 // `distribute` trip count on the host.
2998 Operation *capturedOp =
2999 cast<ComposableOpInterface>(Val: getOperation()).findCapturedOp();
3000 if (auto loopNestOp = dyn_cast_if_present<LoopNestOp>(Val: capturedOp)) {
3001 SmallVector<LoopWrapperInterface> loopWrappers;
3002 loopNestOp.gatherWrappers(wrappers&: loopWrappers);
3003
3004 LoopWrapperInterface *innermostWrapper = loopWrappers.begin();
3005 if (isa<SimdOp>(Val: innermostWrapper))
3006 innermostWrapper = std::next(x: innermostWrapper);
3007
3008 auto numWrappers = std::distance(first: innermostWrapper, last: loopWrappers.end());
3009 if (numWrappers != 1)
3010 return false;
3011
3012 if (!isa<DistributeOp>(Val: innermostWrapper))
3013 return false;
3014
3015 Operation *parentOp = innermostWrapper->getOperation()->getParentOp();
3016 if (isa_and_present<TeamsOp>(Val: parentOp) &&
3017 parentOp->getParentOp() == getOperation())
3018 return true;
3019 }
3020
3021 return false;
3022}
3023
3024/// An `omp.target` `in_reduction` operand is captured by a `map_entries` entry
3025/// when the entry's `MapInfoOp` var_ptr is the same SSA value, or another
3026/// result of the same defining op. At this stage, exact identity can only be
3027/// required for block arguments, which have no defining op. Flang emits
3028/// `hlfir.declare` #0 for the `in_reduction` operand and #1 for the map
3029/// `var_ptr`; these collapse to the same value after lowering, but that cannot
3030/// be enforced here.
3031static bool targetInReductionCapturedBy(Value inReductionVar, Value mapVarPtr) {
3032 if (mapVarPtr == inReductionVar)
3033 return true;
3034 Operation *def = inReductionVar.getDefiningOp();
3035 return def && mapVarPtr.getDefiningOp() == def;
3036}
3037
3038LogicalResult TargetOp::verify() {
3039 if (failed(Result: verifyAllocateClause(
3040 op: getOperation(), allocateVars: getAllocateVars(), allocatorVars: getAllocatorVars(),
3041 allocateAlignments: getAllocateAlignmentsAttr(), allocatePrivateIndices: getAllocatePrivateIndicesAttr(),
3042 privateVars: getPrivateVars(), privateSyms: getPrivateSymsAttr())))
3043 return failure();
3044
3045 if (getKernelType() == TargetExecMode::bare && !isCombined())
3046 return emitOpError() << "bare kernel requires 'omp.combined'";
3047
3048 if (failed(Result: verifyDependVarList(op: *this, dependKinds: getDependKinds(), dependVars: getDependVars(),
3049 iteratedKinds: getDependIteratedKinds(),
3050 iteratedVars: getDependIterated())))
3051 return failure();
3052
3053 if (failed(Result: verifyMapInfoDefinedArgs(op: *this, clauseName: "has_device_addr",
3054 vars: getHasDeviceAddrVars())))
3055 return failure();
3056
3057 if (failed(Result: verifyMapClause(op: *this, mapVars: getMapVars(), mapIterated: getMapIterated())))
3058 return failure();
3059
3060 if (failed(Result: verifyDynGroupprivateClause(
3061 op: *this, accessGroup: getDynGroupprivateAccessGroupAttr(),
3062 fallback: getDynGroupprivateFallbackAttr(), dynGroupprivateSize: getDynGroupprivateSize())))
3063 return failure();
3064
3065 if (failed(Result: verifyPrivateVarList(op&: *this)))
3066 return failure();
3067
3068 if (failed(Result: verifyReductionVarList(op: *this, reductionSyms: getInReductionSyms(),
3069 reductionVars: getInReductionVars(),
3070 reductionByref: getInReductionByref())))
3071 return failure();
3072
3073 // An `in_reduction` operand on `omp.target` has no dedicated entry block
3074 // argument; inside the region it is accessed through the block argument of a
3075 // matching `map_entries` entry, and the host rewrites that map argument to
3076 // the reduction-private storage. Require every `in_reduction` operand to be
3077 // captured by at least one `map_entries` entry.
3078 for (Value inReductionVar : getInReductionVars()) {
3079 bool captured = false;
3080 for (Value mapVar : getMapVars()) {
3081 auto mapInfo = mapVar.getDefiningOp<MapInfoOp>();
3082 if (targetInReductionCapturedBy(inReductionVar, mapVarPtr: mapInfo.getVarPtr())) {
3083 captured = true;
3084 break;
3085 }
3086 }
3087 if (!captured)
3088 return emitOpError() << "in_reduction variable must be captured by a "
3089 "matching map_entries entry";
3090 }
3091
3092 return verifyPrivateVarsMapping(targetOp: *this);
3093}
3094
3095LogicalResult TargetOp::verifyRegions() {
3096 auto teamsOps = getOps<TeamsOp>();
3097 auto numNestedTeams = std::distance(first: teamsOps.begin(), last: teamsOps.end());
3098 if (numNestedTeams > 1)
3099 return emitError(message: "target containing multiple 'omp.teams' nested ops");
3100
3101 if (numNestedTeams == 0) {
3102 switch (getKernelType()) {
3103 case TargetExecMode::bare:
3104 return emitOpError()
3105 << "bare kernel must contain a nested 'omp.teams' operation";
3106 case TargetExecMode::spmd_no_loop:
3107 return emitOpError() << "spmd_no_loop kernel must contain a nested "
3108 "'omp.teams' operation";
3109 default:
3110 break;
3111 }
3112 }
3113
3114 Operation *capturedOp =
3115 cast<ComposableOpInterface>(Val: getOperation()).findCapturedOp();
3116 if ((getKernelType() == TargetExecMode::spmd ||
3117 getKernelType() == TargetExecMode::spmd_no_loop) &&
3118 !isa_and_present<LoopNestOp>(Val: capturedOp))
3119 return emitOpError()
3120 << "SPMD kernel must capture an 'omp.loop_nest' operation";
3121
3122 bool isTargetDevice = false;
3123 if (auto offloadMod = (*this)->getParentOfType<OffloadModuleInterface>())
3124 if (offloadMod.getIsTargetDevice())
3125 isTargetDevice = true;
3126
3127 // Check that host_eval values are only used in legal ways.
3128 llvm::ArrayRef<BlockArgument> hostEvalBlockArgs =
3129 cast<BlockArgOpenMPOpInterface>(Val: getOperation()).getHostEvalBlockArgs();
3130
3131 bool hostEvalTripCount = hasHostEvalTripCount();
3132 for (Value hostEvalArg : hostEvalBlockArgs) {
3133 for (Operation *user : hostEvalArg.getUsers()) {
3134 if (auto teamsOp = dyn_cast<TeamsOp>(Val: user)) {
3135 // Check if used in num_teams_lower or any of num_teams_upper_vars
3136 if (hostEvalArg == teamsOp.getNumTeamsLower() ||
3137 llvm::is_contained(Range: teamsOp.getNumTeamsUpperVars(), Element: hostEvalArg) ||
3138 llvm::is_contained(Range: teamsOp.getThreadLimitVars(), Element: hostEvalArg))
3139 continue;
3140
3141 return emitOpError() << "host_eval argument only legal as 'num_teams' "
3142 "and 'thread_limit' in 'omp.teams'";
3143 }
3144 if (auto parallelOp = dyn_cast<ParallelOp>(Val: user)) {
3145 if (llvm::is_contained(Range: parallelOp.getNumThreadsVars(), Element: hostEvalArg))
3146 continue;
3147
3148 return emitOpError()
3149 << "host_eval argument only legal as 'num_threads' in "
3150 "'omp.parallel'";
3151 }
3152 if (auto loopNestOp = dyn_cast<LoopNestOp>(Val: user)) {
3153 if (hostEvalTripCount &&
3154 (llvm::is_contained(Range: loopNestOp.getLoopLowerBounds(), Element: hostEvalArg) ||
3155 llvm::is_contained(Range: loopNestOp.getLoopUpperBounds(), Element: hostEvalArg) ||
3156 llvm::is_contained(Range: loopNestOp.getLoopSteps(), Element: hostEvalArg)))
3157 continue;
3158
3159 return emitOpError() << "host_eval argument only legal as loop bounds "
3160 "and steps in 'omp.loop_nest' when trip count "
3161 "must be evaluated in the host";
3162 }
3163
3164 return emitOpError() << "host_eval argument illegal use in '"
3165 << user->getName() << "' operation";
3166 }
3167 }
3168
3169 if (hostEvalTripCount && !isTargetDevice) {
3170 auto loopOp = cast<LoopNestOp>(Val: capturedOp);
3171 for (auto arg : llvm::concat<Value>(Ranges: loopOp.getLoopLowerBounds(),
3172 Ranges: loopOp.getLoopUpperBounds(),
3173 Ranges: loopOp.getLoopSteps())) {
3174 if (!llvm::is_contained(Range&: hostEvalBlockArgs, Element: arg))
3175 return emitOpError() << "nested 'omp.loop_nest' bounds expected to "
3176 "be host-evaluated";
3177 }
3178 }
3179
3180 return success();
3181}
3182
3183//===----------------------------------------------------------------------===//
3184// ParallelOp
3185//===----------------------------------------------------------------------===//
3186
3187void ParallelOp::build(OpBuilder &builder, OperationState &state,
3188 ArrayRef<NamedAttribute> attributes) {
3189 ParallelOp::build(odsBuilder&: builder, odsState&: state, /*allocate_vars=*/ValueRange(),
3190 /*allocator_vars=*/ValueRange(),
3191 /*allocate_alignments=*/nullptr,
3192 /*allocate_private_indices=*/nullptr, /*if_expr=*/nullptr,
3193 /*num_threads_vars=*/ValueRange(),
3194 /*private_vars=*/ValueRange(),
3195 /*private_syms=*/nullptr, /*private_needs_barrier=*/false,
3196 /*proc_bind_kind=*/nullptr,
3197 /*reduction_mods =*/nullptr,
3198 /*reduction_vars=*/ValueRange(),
3199 /*reduction_byref=*/nullptr, /*reduction_syms=*/nullptr);
3200 state.addAttributes(newAttributes: attributes);
3201}
3202
3203void ParallelOp::build(OpBuilder &builder, OperationState &state,
3204 const ParallelOperands &clauses) {
3205 MLIRContext *ctx = builder.getContext();
3206 ParallelOp::build(odsBuilder&: builder, odsState&: state, allocate_vars: clauses.allocateVars, allocator_vars: clauses.allocatorVars,
3207 allocate_alignments: makeDenseI64ArrayAttr(ctx, intArray: clauses.allocateAlignments),
3208 allocate_private_indices: makeDenseI64ArrayAttr(ctx, intArray: clauses.allocatePrivateIndices),
3209 if_expr: clauses.ifExpr, num_threads_vars: clauses.numThreadsVars, private_vars: clauses.privateVars,
3210 private_syms: makeArrayAttr(context: ctx, attrs: clauses.privateSyms),
3211 private_needs_barrier: clauses.privateNeedsBarrier, proc_bind_kind: clauses.procBindKind,
3212 reduction_mods: makeArrayAttr(context: ctx, attrs: clauses.reductionMods),
3213 reduction_vars: clauses.reductionVars,
3214 reduction_byref: makeDenseBoolArrayAttr(ctx, boolArray: clauses.reductionByref),
3215 reduction_syms: makeArrayAttr(context: ctx, attrs: clauses.reductionSyms));
3216}
3217
3218template <typename OpType>
3219static LogicalResult verifyPrivateVarList(OpType &op) {
3220 auto privateVars = op.getPrivateVars();
3221 auto privateSyms = op.getPrivateSymsAttr();
3222
3223 if (privateVars.empty() && (privateSyms == nullptr || privateSyms.empty()))
3224 return success();
3225
3226 auto numPrivateVars = privateVars.size();
3227 auto numPrivateSyms = (privateSyms == nullptr) ? 0 : privateSyms.size();
3228
3229 if (numPrivateVars != numPrivateSyms)
3230 return op.emitError() << "inconsistent number of private variables and "
3231 "privatizer op symbols, private vars: "
3232 << numPrivateVars
3233 << " vs. privatizer op symbols: " << numPrivateSyms;
3234
3235 for (auto privateVarInfo : llvm::zip_equal(privateVars, privateSyms)) {
3236 Type varType = std::get<0>(privateVarInfo).getType();
3237 SymbolRefAttr privateSym = cast<SymbolRefAttr>(std::get<1>(privateVarInfo));
3238 PrivateClauseOp privatizerOp =
3239 SymbolTable::lookupNearestSymbolFrom<PrivateClauseOp>(op, privateSym);
3240
3241 if (privatizerOp == nullptr)
3242 return op.emitError() << "failed to lookup privatizer op with symbol: '"
3243 << privateSym << "'";
3244
3245 Type privatizerType = privatizerOp.getArgType();
3246
3247 if (privatizerType && (varType != privatizerType))
3248 return op.emitError()
3249 << "type mismatch between a "
3250 << stringifyDataSharingClauseType(
3251 val: privatizerOp.getDataSharingType())
3252 << " variable and its privatizer op, var type: " << varType
3253 << " vs. privatizer op type: " << privatizerType;
3254
3255 if (privatizerOp.getDataSharingType() == DataSharingClauseType::Linear)
3256 return op.emitError() << "`linear` privatizer '" << privateSym
3257 << "' can only be used in a `linear` clause";
3258 }
3259
3260 return success();
3261}
3262
3263/// Verifies that each linear variable has a `linear` privatizer.
3264template <typename OpType>
3265static LogicalResult verifyLinearSyms(OpType op) {
3266 OperandRange linearVars = op.getLinearVars();
3267 std::optional<ArrayAttr> linearSyms = op.getLinearSyms();
3268 if (linearVars.empty() && (!linearSyms || linearSyms->empty()))
3269 return success();
3270
3271 if (!linearSyms || linearSyms->size() != linearVars.size())
3272 return op.emitOpError()
3273 << "expected as many linear privatizers as linear variables";
3274
3275 for (auto [var, sym] : llvm::zip_equal(t&: linearVars, u&: *linearSyms)) {
3276 auto symRef = cast<SymbolRefAttr>(Val: sym);
3277 auto privatizer =
3278 SymbolTable::lookupNearestSymbolFrom<PrivateClauseOp>(op, symRef);
3279 if (!privatizer)
3280 return op.emitOpError() << "failed to lookup privatizer op with symbol: '"
3281 << symRef << "'";
3282 if (privatizer.getDataSharingType() != DataSharingClauseType::Linear)
3283 return op.emitOpError() << "privatizer '" << symRef
3284 << "' of a linear variable must be `linear`";
3285 Type argType = privatizer.getArgType();
3286 if (argType && argType != var.getType())
3287 return op.emitOpError()
3288 << "type mismatch between a linear variable and its privatizer "
3289 "op, var type: "
3290 << var.getType() << " vs. privatizer op type: " << argType;
3291 }
3292 return success();
3293}
3294
3295LogicalResult ParallelOp::verify() {
3296 if (failed(Result: verifyPrivateVarList(op&: *this)))
3297 return failure();
3298 if (failed(Result: verifyAllocateClause(
3299 op: getOperation(), allocateVars: getAllocateVars(), allocatorVars: getAllocatorVars(),
3300 allocateAlignments: getAllocateAlignmentsAttr(), allocatePrivateIndices: getAllocatePrivateIndicesAttr(),
3301 privateVars: getPrivateVars(), privateSyms: getPrivateSymsAttr(),
3302 /*requirePrivateIndices=*/true)))
3303 return failure();
3304
3305 return verifyReductionVarList(op: *this, reductionSyms: getReductionSyms(), reductionVars: getReductionVars(),
3306 reductionByref: getReductionByref(), reductionMods: getReductionModsAttr());
3307}
3308
3309LogicalResult ParallelOp::verifyRegions() {
3310 auto distChildOps = getOps<DistributeOp>();
3311 int numDistChildOps = std::distance(first: distChildOps.begin(), last: distChildOps.end());
3312 if (numDistChildOps > 1)
3313 return emitError()
3314 << "multiple 'omp.distribute' nested inside of 'omp.parallel'";
3315
3316 if (numDistChildOps == 1) {
3317 if (!isComposite())
3318 return emitError()
3319 << "'omp.composite' attribute missing from composite operation";
3320
3321 auto *ompDialect = getContext()->getLoadedDialect<OpenMPDialect>();
3322 Operation &distributeOp = **distChildOps.begin();
3323 for (Operation &childOp : getOps()) {
3324 if (&childOp == &distributeOp || ompDialect != childOp.getDialect())
3325 continue;
3326
3327 if (!childOp.hasTrait<OpTrait::IsTerminator>())
3328 return emitError() << "unexpected OpenMP operation inside of composite "
3329 "'omp.parallel': "
3330 << childOp.getName();
3331 }
3332 } else if (isComposite()) {
3333 return emitError()
3334 << "'omp.composite' attribute present in non-composite operation";
3335 }
3336 return success();
3337}
3338
3339//===----------------------------------------------------------------------===//
3340// TeamsOp
3341//===----------------------------------------------------------------------===//
3342
3343static bool opInGlobalImplicitParallelRegion(Operation *op) {
3344 while ((op = op->getParentOp()))
3345 if (isa<OpenMPDialect>(Val: op->getDialect()))
3346 return false;
3347 return true;
3348}
3349
3350void TeamsOp::build(OpBuilder &builder, OperationState &state,
3351 const TeamsOperands &clauses) {
3352 MLIRContext *ctx = builder.getContext();
3353 // TODO Store clauses in op: privateVars, privateSyms, privateNeedsBarrier
3354 TeamsOp::build(
3355 odsBuilder&: builder, odsState&: state, allocate_vars: clauses.allocateVars, allocator_vars: clauses.allocatorVars,
3356 allocate_alignments: makeDenseI64ArrayAttr(ctx, intArray: clauses.allocateAlignments),
3357 allocate_private_indices: makeDenseI64ArrayAttr(ctx, intArray: clauses.allocatePrivateIndices),
3358 dyn_groupprivate_access_group: clauses.dynGroupprivateAccessGroup, dyn_groupprivate_fallback: clauses.dynGroupprivateFallback,
3359 dyn_groupprivate_size: clauses.dynGroupprivateSize, if_expr: clauses.ifExpr, num_teams_lower: clauses.numTeamsLower,
3360 num_teams_upper_vars: clauses.numTeamsUpperVars, /*private_vars=*/{}, /*private_syms=*/nullptr,
3361 /*private_needs_barrier=*/false,
3362 reduction_mods: makeArrayAttr(context: ctx, attrs: clauses.reductionMods), reduction_vars: clauses.reductionVars,
3363 reduction_byref: makeDenseBoolArrayAttr(ctx, boolArray: clauses.reductionByref),
3364 reduction_syms: makeArrayAttr(context: ctx, attrs: clauses.reductionSyms), thread_limit_vars: clauses.threadLimitVars);
3365}
3366
3367// Verify num_teams clause
3368static LogicalResult verifyNumTeamsClause(Operation *op, Value numTeamsLower,
3369 OperandRange numTeamsUpperVars) {
3370 // If lower is specified, upper must have exactly one value
3371 if (numTeamsLower) {
3372 if (numTeamsUpperVars.size() != 1)
3373 return op->emitError(
3374 message: "expected exactly one num_teams upper bound when lower bound is "
3375 "specified");
3376 if (numTeamsLower.getType() != numTeamsUpperVars[0].getType())
3377 return op->emitError(
3378 message: "expected num_teams upper bound and lower bound to be "
3379 "the same type");
3380 }
3381
3382 return success();
3383}
3384
3385LogicalResult TeamsOp::verify() {
3386 // Check parent region
3387 // TODO If nested inside of a target region, also check that it does not
3388 // contain any statements, declarations or directives other than this
3389 // omp.teams construct. The issue is how to support the initialization of
3390 // this operation's own arguments (allow SSA values across omp.target?).
3391 Operation *op = getOperation();
3392 auto parentTarget = llvm::dyn_cast_if_present<TargetOp>(Val: op->getParentOp());
3393 if (!parentTarget && !opInGlobalImplicitParallelRegion(op))
3394 return emitError(message: "expected to be nested inside of omp.target or not nested "
3395 "in any OpenMP dialect operations");
3396
3397 // Check for num_teams clause restrictions
3398 if (failed(Result: verifyNumTeamsClause(op, numTeamsLower: this->getNumTeamsLower(),
3399 numTeamsUpperVars: this->getNumTeamsUpperVars())))
3400 return failure();
3401
3402 if (parentTarget &&
3403 parentTarget.getKernelType() == TargetExecMode::spmd_no_loop &&
3404 (getNumTeamsLower() || !getNumTeamsUpperVars().empty()))
3405 return emitOpError() << "'num_teams' not allowed in SPMD-no-loop kernels";
3406
3407 if (failed(Result: verifyAllocateClause(
3408 op: getOperation(), allocateVars: getAllocateVars(), allocatorVars: getAllocatorVars(),
3409 allocateAlignments: getAllocateAlignmentsAttr(), allocatePrivateIndices: getAllocatePrivateIndicesAttr(),
3410 privateVars: getPrivateVars(), privateSyms: getPrivateSymsAttr())))
3411 return failure();
3412
3413 if (failed(Result: verifyDynGroupprivateClause(
3414 op, accessGroup: getDynGroupprivateAccessGroupAttr(),
3415 fallback: getDynGroupprivateFallbackAttr(), dynGroupprivateSize: getDynGroupprivateSize())))
3416 return failure();
3417
3418 if (failed(Result: verifyPrivateVarList(op&: *this)))
3419 return failure();
3420
3421 return verifyReductionVarList(op: *this, reductionSyms: getReductionSyms(), reductionVars: getReductionVars(),
3422 reductionByref: getReductionByref(), reductionMods: getReductionModsAttr());
3423}
3424
3425//===----------------------------------------------------------------------===//
3426// SectionOp
3427//===----------------------------------------------------------------------===//
3428
3429OperandRange SectionOp::getPrivateVars() {
3430 return getParentOp().getPrivateVars();
3431}
3432
3433OperandRange SectionOp::getReductionVars() {
3434 return getParentOp().getReductionVars();
3435}
3436
3437//===----------------------------------------------------------------------===//
3438// SectionsOp
3439//===----------------------------------------------------------------------===//
3440
3441void SectionsOp::build(OpBuilder &builder, OperationState &state,
3442 const SectionsOperands &clauses) {
3443 MLIRContext *ctx = builder.getContext();
3444 // TODO Store clauses in op: privateVars, privateSyms, privateNeedsBarrier
3445 SectionsOp::build(odsBuilder&: builder, odsState&: state, allocate_vars: clauses.allocateVars, allocator_vars: clauses.allocatorVars,
3446 allocate_alignments: makeDenseI64ArrayAttr(ctx, intArray: clauses.allocateAlignments),
3447 allocate_private_indices: makeDenseI64ArrayAttr(ctx, intArray: clauses.allocatePrivateIndices),
3448 nowait: clauses.nowait, /*private_vars=*/{},
3449 /*private_syms=*/nullptr, /*private_needs_barrier=*/nullptr,
3450 /*private_assume_bounded_recipe_effects=*/nullptr,
3451 reduction_mods: makeArrayAttr(context: ctx, attrs: clauses.reductionMods),
3452 reduction_vars: clauses.reductionVars,
3453 reduction_byref: makeDenseBoolArrayAttr(ctx, boolArray: clauses.reductionByref),
3454 reduction_syms: makeArrayAttr(context: ctx, attrs: clauses.reductionSyms),
3455 /*reduction_assume_bounded_recipe_effects=*/nullptr);
3456}
3457
3458LogicalResult SectionsOp::verify() {
3459 if (isCombined())
3460 return emitOpError() << "cannot be a non-innermost combined construct leaf";
3461
3462 if (failed(Result: verifyAllocateClause(
3463 op: getOperation(), allocateVars: getAllocateVars(), allocatorVars: getAllocatorVars(),
3464 allocateAlignments: getAllocateAlignmentsAttr(), allocatePrivateIndices: getAllocatePrivateIndicesAttr(),
3465 privateVars: getPrivateVars(), privateSyms: getPrivateSymsAttr())))
3466 return failure();
3467
3468 return verifyReductionVarList(op: *this, reductionSyms: getReductionSyms(), reductionVars: getReductionVars(),
3469 reductionByref: getReductionByref(), reductionMods: getReductionModsAttr());
3470}
3471
3472LogicalResult SectionsOp::verifyRegions() {
3473 for (auto &inst : *getRegion().begin()) {
3474 if (!(isa<SectionOp>(Val: inst) || isa<TerminatorOp>(Val: inst))) {
3475 return emitOpError()
3476 << "expected omp.section op or terminator op inside region";
3477 }
3478 }
3479
3480 return success();
3481}
3482
3483/// Add arbitrary read, write, allocate, and free effects.
3484static void addUnknownMemoryEffects(
3485 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
3486 effects.emplace_back(Args: MemoryEffects::Read::get());
3487 effects.emplace_back(Args: MemoryEffects::Write::get());
3488 effects.emplace_back(Args: MemoryEffects::Allocate::get());
3489 effects.emplace_back(Args: MemoryEffects::Free::get());
3490}
3491
3492/// Privatisation may read the original as a mold or firstprivate copy source.
3493/// Any lastprivate copy-back is explicit in the nested operations.
3494static void getPrivatisationEffects(
3495 Operation *op, SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
3496 auto iface = dyn_cast<BlockArgOpenMPOpInterface>(Val: op);
3497
3498 OperandRange sourceVars = iface.getPrivateVars();
3499 for (OpOperand &source :
3500 MutableArrayRef(sourceVars.getBase(), sourceVars.size()))
3501 effects.emplace_back(Args: MemoryEffects::Read::get(), Args: &source);
3502 for (BlockArgument priv : iface.getPrivateBlockArgs()) {
3503 effects.emplace_back(Args: MemoryEffects::Write::get(), Args&: priv);
3504 effects.emplace_back(Args: MemoryEffects::Allocate::get(), Args&: priv);
3505 effects.emplace_back(Args: MemoryEffects::Free::get(), Args&: priv);
3506 }
3507}
3508
3509/// Reductions initialise private accumulators and combine them into the
3510/// original variables. Account for reduction and in_reduction clauses.
3511static void
3512getReductionEffects(Operation *op,
3513 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
3514 auto iface = dyn_cast<BlockArgOpenMPOpInterface>(Val: op);
3515
3516 auto addEffects = [&](OperandRange sourceVars,
3517 ArrayRef<BlockArgument> privateArgs) {
3518 for (OpOperand &source :
3519 MutableArrayRef(sourceVars.getBase(), sourceVars.size())) {
3520 effects.emplace_back(Args: MemoryEffects::Read::get(), Args: &source);
3521 effects.emplace_back(Args: MemoryEffects::Write::get(), Args: &source);
3522 }
3523
3524 for (BlockArgument priv : privateArgs) {
3525 effects.emplace_back(Args: MemoryEffects::Allocate::get(), Args&: priv);
3526 effects.emplace_back(Args: MemoryEffects::Free::get(), Args&: priv);
3527 effects.emplace_back(Args: MemoryEffects::Read::get(), Args&: priv);
3528 effects.emplace_back(Args: MemoryEffects::Write::get(), Args&: priv);
3529 }
3530 };
3531
3532 addEffects(iface.getReductionVars(), iface.getReductionBlockArgs());
3533 addEffects(iface.getInReductionVars(), iface.getInReductionBlockArgs());
3534}
3535
3536static void getBarrierEffects(
3537 llvm::SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
3538 &effects) {
3539 effects.emplace_back(Args: MemoryEffects::Read::get());
3540 effects.emplace_back(Args: MemoryEffects::Write::get());
3541}
3542
3543void SectionsOp::getEffects(
3544 llvm::SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
3545 &effects) {
3546 // The presence of the nowait clause eliminates the implicit barrier at the
3547 // end of a sections construct
3548 if (!getNowait()) {
3549 getBarrierEffects(effects);
3550 }
3551
3552 bool hasUnknownRecipeEffects =
3553 (!getPrivateVars().empty() && !getPrivateAssumeBoundedRecipeEffects()) ||
3554 (!getReductionVars().empty() &&
3555 !getReductionAssumeBoundedRecipeEffects());
3556
3557 if (hasUnknownRecipeEffects) {
3558 addUnknownMemoryEffects(effects);
3559 } else {
3560 getPrivatisationEffects(op: *this, effects);
3561 getReductionEffects(op: *this, effects);
3562 }
3563}
3564
3565//===----------------------------------------------------------------------===//
3566// ScopeOp
3567//===----------------------------------------------------------------------===//
3568
3569void ScopeOp::build(OpBuilder &builder, OperationState &state,
3570 const ScopeOperands &clauses) {
3571 MLIRContext *ctx = builder.getContext();
3572 ScopeOp::build(
3573 odsBuilder&: builder, odsState&: state, allocate_vars: clauses.allocateVars, allocator_vars: clauses.allocatorVars,
3574 allocate_alignments: makeDenseI64ArrayAttr(ctx, intArray: clauses.allocateAlignments),
3575 allocate_private_indices: makeDenseI64ArrayAttr(ctx, intArray: clauses.allocatePrivateIndices),
3576 nowait: clauses.nowait, private_vars: clauses.privateVars,
3577 private_syms: makeArrayAttr(context: ctx, attrs: clauses.privateSyms), private_needs_barrier: clauses.privateNeedsBarrier,
3578 reduction_mods: makeArrayAttr(context: ctx, attrs: clauses.reductionMods), reduction_vars: clauses.reductionVars,
3579 reduction_byref: makeDenseBoolArrayAttr(ctx, boolArray: clauses.reductionByref),
3580 reduction_syms: makeArrayAttr(context: ctx, attrs: clauses.reductionSyms));
3581}
3582
3583LogicalResult ScopeOp::verify() {
3584 if (failed(Result: verifyAllocateClause(
3585 op: getOperation(), allocateVars: getAllocateVars(), allocatorVars: getAllocatorVars(),
3586 allocateAlignments: getAllocateAlignmentsAttr(), allocatePrivateIndices: getAllocatePrivateIndicesAttr(),
3587 privateVars: getPrivateVars(), privateSyms: getPrivateSymsAttr(),
3588 /*requirePrivateIndices=*/true)))
3589 return failure();
3590
3591 if (failed(Result: verifyPrivateVarList(op&: *this)))
3592 return failure();
3593
3594 return verifyReductionVarList(op: *this, reductionSyms: getReductionSyms(), reductionVars: getReductionVars(),
3595 reductionByref: getReductionByref(), reductionMods: getReductionModsAttr());
3596}
3597
3598//===----------------------------------------------------------------------===//
3599// SingleOp
3600//===----------------------------------------------------------------------===//
3601
3602void SingleOp::build(OpBuilder &builder, OperationState &state,
3603 const SingleOperands &clauses) {
3604 MLIRContext *ctx = builder.getContext();
3605 // TODO Store clauses in op: privateVars, privateSyms, privateNeedsBarrier
3606 SingleOp::build(
3607 odsBuilder&: builder, odsState&: state, allocate_vars: clauses.allocateVars, allocator_vars: clauses.allocatorVars,
3608 allocate_alignments: makeDenseI64ArrayAttr(ctx, intArray: clauses.allocateAlignments),
3609 allocate_private_indices: makeDenseI64ArrayAttr(ctx, intArray: clauses.allocatePrivateIndices),
3610 copyprivate_vars: clauses.copyprivateVars, copyprivate_syms: makeArrayAttr(context: ctx, attrs: clauses.copyprivateSyms),
3611 /*copyprivate_assume_bounded_recipe_effects=*/nullptr, nowait: clauses.nowait,
3612 /*private_vars=*/{}, /*private_syms=*/nullptr,
3613 /*private_needs_barrier=*/nullptr,
3614 /*private_assume_bounded_recipe_effects=*/nullptr);
3615}
3616
3617LogicalResult SingleOp::verify() {
3618 if (failed(Result: verifyAllocateClause(
3619 op: getOperation(), allocateVars: getAllocateVars(), allocatorVars: getAllocatorVars(),
3620 allocateAlignments: getAllocateAlignmentsAttr(), allocatePrivateIndices: getAllocatePrivateIndicesAttr(),
3621 privateVars: getPrivateVars(), privateSyms: getPrivateSymsAttr())))
3622 return failure();
3623
3624 return verifyCopyprivateVarList(op: *this, copyprivateVars: getCopyprivateVars(),
3625 copyprivateSyms: getCopyprivateSyms());
3626}
3627
3628void SingleOp::getEffects(
3629 llvm::SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
3630 &effects) {
3631 // The presence of the nowait clause eliminates the implicit barrier at the
3632 // end of a single construct.
3633 if (!getNowait()) {
3634 getBarrierEffects(effects);
3635 }
3636
3637 bool hasUnknownRecipeEffects =
3638 (!getPrivateVars().empty() && !getPrivateAssumeBoundedRecipeEffects()) ||
3639 (!getCopyprivateVars().empty() &&
3640 !getCopyprivateAssumeBoundedRecipeEffects());
3641 if (hasUnknownRecipeEffects) {
3642 addUnknownMemoryEffects(effects);
3643 } else {
3644 getPrivatisationEffects(op: *this, effects);
3645 // The unknown executing thread supplies the value and the other threads
3646 // receive it, so each copyprivate operand may be both read and written.
3647 OperandRange copyprivateVars = getCopyprivateVars();
3648 for (OpOperand &variable :
3649 MutableArrayRef(copyprivateVars.getBase(), copyprivateVars.size())) {
3650 effects.emplace_back(Args: MemoryEffects::Read::get(), Args: &variable);
3651 effects.emplace_back(Args: MemoryEffects::Write::get(), Args: &variable);
3652 }
3653 }
3654}
3655
3656//===----------------------------------------------------------------------===//
3657// WorkshareOp
3658//===----------------------------------------------------------------------===//
3659
3660void WorkshareOp::build(OpBuilder &builder, OperationState &state,
3661 const WorkshareOperands &clauses) {
3662 WorkshareOp::build(odsBuilder&: builder, odsState&: state, nowait: clauses.nowait);
3663}
3664
3665LogicalResult WorkshareOp::verify() {
3666 if (isCombined())
3667 return emitOpError() << "cannot be a non-innermost combined construct leaf";
3668
3669 return success();
3670}
3671
3672//===----------------------------------------------------------------------===//
3673// WorkshareLoopWrapperOp
3674//===----------------------------------------------------------------------===//
3675
3676LogicalResult WorkshareLoopWrapperOp::verifyRegions() {
3677 if (isa_and_nonnull<LoopWrapperInterface>(Val: (*this)->getParentOp()) ||
3678 getNestedWrapper())
3679 return emitOpError() << "expected to be a standalone loop wrapper";
3680
3681 return success();
3682}
3683
3684//===----------------------------------------------------------------------===//
3685// LoopWrapperInterface
3686//===----------------------------------------------------------------------===//
3687
3688LogicalResult LoopWrapperInterface::verifyImpl() {
3689 Operation *op = this->getOperation();
3690 if (!op->hasTrait<OpTrait::NoTerminator>() ||
3691 !op->hasTrait<OpTrait::SingleBlock>())
3692 return emitOpError() << "loop wrapper must also have the `NoTerminator` "
3693 "and `SingleBlock` traits";
3694
3695 if (op->getNumRegions() != 1)
3696 return emitOpError() << "loop wrapper does not contain exactly one region";
3697
3698 Region &region = op->getRegion(index: 0);
3699 if (range_size(Range: region.getOps()) != 1)
3700 return emitOpError()
3701 << "loop wrapper does not contain exactly one nested op";
3702
3703 Operation &firstOp = *region.op_begin();
3704 if (!isa<LoopNestOp, LoopWrapperInterface>(Val: firstOp))
3705 return emitOpError() << "nested in loop wrapper is not another loop "
3706 "wrapper or `omp.loop_nest`";
3707
3708 return success();
3709}
3710
3711//===----------------------------------------------------------------------===//
3712// ComposableOpInterface
3713//===----------------------------------------------------------------------===//
3714
3715Operation *ComposableOpInterface::findCapturedOp() {
3716 Operation *op = this->getOperation();
3717
3718 // Handle the composite case by returning the wrapped omp.loop_nest.
3719 if (auto wrapperOp = dyn_cast<LoopWrapperInterface>(Val: op))
3720 return wrapperOp.getWrappedLoop();
3721
3722 // Do not look further if this op is not combined with any of its children.
3723 // Need to check for composite for the omp.parallel case, which is not a loop
3724 // wrapper itself.
3725 if (!isCombined() && !isComposite())
3726 return op;
3727
3728 Region &region = op->getRegion(index: 0);
3729 for (Operation &nestedOp : region.getOps()) {
3730 if (auto wrapperOp = dyn_cast<LoopWrapperInterface>(Val: &nestedOp))
3731 return wrapperOp.getWrappedLoop();
3732
3733 if (auto composableOp = dyn_cast<ComposableOpInterface>(Val: &nestedOp))
3734 return composableOp.findCapturedOp();
3735 }
3736
3737 // This can only be reached if the op has an omp.combined attribute but the
3738 // corresponding nested composable op has been deleted. In that case, it's
3739 // correct to return this operation.
3740 return op;
3741}
3742
3743LogicalResult ComposableOpInterface::verifyImpl() {
3744 Operation *op = this->getOperation();
3745
3746 if (op->getNumRegions() != 1)
3747 return emitOpError() << "composable ops must have a single region";
3748
3749 if (isComposite() && !isa<LoopWrapperInterface, ParallelOp>(Val: op))
3750 return emitOpError() << "non-loop wrapper cannot be composite";
3751
3752 // If combined, must have exactly one eligible nested op (composable or loop
3753 // wrapper).
3754 if (isCombined()) {
3755 Operation *nestedOp = nullptr;
3756 auto count = llvm::count_if(
3757 Range: op->getRegion(index: 0).getOps(), P: [&nestedOp](mlir::Operation &op) {
3758 if (isa<ComposableOpInterface, LoopWrapperInterface>(Val: op)) {
3759 nestedOp = &op;
3760 return true;
3761 }
3762 return false;
3763 });
3764
3765 // Make an exception for ops marked as omp.combined with no eligible nested
3766 // ops: this situation should be disallowed, but it can be reached if an
3767 // MLIR optimization pass find that the child operation has no side effects
3768 // (many ComposableOpInterface ops have RecursiveMemoryEffects), so it gets
3769 // deleted without updating the parent's attribute.
3770 //
3771 // Since there's a well defined way of handling that situation (treat it as
3772 // non-combined), we relax the requirement here. Ensuring the parent is
3773 // updated every time a pass that can potentially remove a child composable
3774 // op runs is less preferable as a solution.
3775 if (count == 0)
3776 return success();
3777
3778 if (count > 1)
3779 return emitOpError()
3780 << "multiple eligible child ops found in combined op";
3781
3782 // This operation cannot be combined if its captured nested op can be
3783 // executed more than once (i.e. its block's successors can reach it) or if
3784 // it's not guaranteed to be executed before all exits of the region (i.e.
3785 // it doesn't dominate all blocks with no successors reachable from the
3786 // entry block).
3787 DominanceInfo domInfo;
3788 Block *parentBlock = nestedOp->getBlock();
3789
3790 for (Block *successor : parentBlock->getSuccessors())
3791 if (successor->isReachable(other: parentBlock))
3792 return emitOpError() << "nested combined child op is part of a loop";
3793
3794 for (Block &block : op->getRegion(index: 0))
3795 if (domInfo.isReachableFromEntry(a: &block) && block.hasNoSuccessors() &&
3796 !domInfo.dominates(a: parentBlock, b: &block))
3797 return emitOpError()
3798 << "nested combined child op doesn't unconditionally execute";
3799 }
3800 return success();
3801}
3802
3803//===----------------------------------------------------------------------===//
3804// LoopOp
3805//===----------------------------------------------------------------------===//
3806
3807void LoopOp::build(OpBuilder &builder, OperationState &state,
3808 const LoopOperands &clauses) {
3809 MLIRContext *ctx = builder.getContext();
3810
3811 LoopOp::build(odsBuilder&: builder, odsState&: state, bind_kind: clauses.bindKind, private_vars: clauses.privateVars,
3812 private_syms: makeArrayAttr(context: ctx, attrs: clauses.privateSyms),
3813 private_needs_barrier: clauses.privateNeedsBarrier, order: clauses.order, order_mod: clauses.orderMod,
3814 reduction_mods: makeArrayAttr(context: ctx, attrs: clauses.reductionMods),
3815 reduction_vars: clauses.reductionVars,
3816 reduction_byref: makeDenseBoolArrayAttr(ctx, boolArray: clauses.reductionByref),
3817 reduction_syms: makeArrayAttr(context: ctx, attrs: clauses.reductionSyms));
3818}
3819
3820LogicalResult LoopOp::verify() {
3821 if (failed(Result: verifyPrivateVarList(op&: *this)))
3822 return failure();
3823
3824 return verifyReductionVarList(op: *this, reductionSyms: getReductionSyms(), reductionVars: getReductionVars(),
3825 reductionByref: getReductionByref(), reductionMods: getReductionModsAttr());
3826}
3827
3828LogicalResult LoopOp::verifyRegions() {
3829 if (llvm::isa_and_nonnull<LoopWrapperInterface>(Val: (*this)->getParentOp()) ||
3830 getNestedWrapper())
3831 return emitOpError() << "expected to be a standalone loop wrapper";
3832
3833 return success();
3834}
3835
3836//===----------------------------------------------------------------------===//
3837// WsloopOp
3838//===----------------------------------------------------------------------===//
3839
3840void WsloopOp::build(OpBuilder &builder, OperationState &state,
3841 ArrayRef<NamedAttribute> attributes) {
3842 build(odsBuilder&: builder, odsState&: state, /*allocate_vars=*/{}, /*allocator_vars=*/{},
3843 /*allocate_alignments=*/nullptr,
3844 /*allocate_private_indices=*/nullptr,
3845 /*linear_vars=*/ValueRange(), /*linear_step_vars=*/ValueRange(),
3846 /*linear_var_types*/ nullptr, /*linear_modifiers=*/nullptr,
3847 /*linear_syms=*/nullptr, /*nowait=*/false, /*order=*/nullptr,
3848 /*order_mod=*/nullptr,
3849 /*ordered=*/nullptr, /*private_vars=*/{}, /*private_syms=*/nullptr,
3850 /*private_needs_barrier=*/false,
3851 /*reduction_mods=*/nullptr, /*reduction_vars=*/ValueRange(),
3852 /*reduction_byref=*/nullptr,
3853 /*reduction_syms=*/nullptr, /*schedule_kind=*/nullptr,
3854 /*schedule_chunk=*/nullptr, /*schedule_mod=*/nullptr,
3855 /*schedule_simd=*/false);
3856 state.addAttributes(newAttributes: attributes);
3857}
3858
3859void WsloopOp::build(OpBuilder &builder, OperationState &state,
3860 const WsloopOperands &clauses) {
3861 MLIRContext *ctx = builder.getContext();
3862 WsloopOp::build(
3863 odsBuilder&: builder, odsState&: state, allocate_vars: clauses.allocateVars, allocator_vars: clauses.allocatorVars,
3864 allocate_alignments: makeDenseI64ArrayAttr(ctx, intArray: clauses.allocateAlignments),
3865 allocate_private_indices: makeDenseI64ArrayAttr(ctx, intArray: clauses.allocatePrivateIndices),
3866 linear_vars: clauses.linearVars, linear_step_vars: clauses.linearStepVars, linear_var_types: clauses.linearVarTypes,
3867 linear_modifiers: clauses.linearModifiers, linear_syms: makeArrayAttr(context: ctx, attrs: clauses.linearSyms),
3868 nowait: clauses.nowait, order: clauses.order, order_mod: clauses.orderMod, ordered: clauses.ordered,
3869 private_vars: clauses.privateVars, private_syms: makeArrayAttr(context: ctx, attrs: clauses.privateSyms),
3870 private_needs_barrier: clauses.privateNeedsBarrier, reduction_mods: makeArrayAttr(context: ctx, attrs: clauses.reductionMods),
3871 reduction_vars: clauses.reductionVars,
3872 reduction_byref: makeDenseBoolArrayAttr(ctx, boolArray: clauses.reductionByref),
3873 reduction_syms: makeArrayAttr(context: ctx, attrs: clauses.reductionSyms), schedule_kind: clauses.scheduleKind,
3874 schedule_chunk: clauses.scheduleChunk, schedule_mod: clauses.scheduleMod, schedule_simd: clauses.scheduleSimd);
3875}
3876
3877LogicalResult WsloopOp::verify() {
3878 if (failed(Result: verifyAllocateClause(
3879 op: getOperation(), allocateVars: getAllocateVars(), allocatorVars: getAllocatorVars(),
3880 allocateAlignments: getAllocateAlignmentsAttr(), allocatePrivateIndices: getAllocatePrivateIndicesAttr(),
3881 privateVars: getPrivateVars(), privateSyms: getPrivateSymsAttr())))
3882 return failure();
3883
3884 if (failed(
3885 Result: verifyLinearModifiers(op: *this, linearModifiers: getLinearModifiers(), linearVars: getLinearVars())))
3886 return failure();
3887 if (getLinearVars().size() &&
3888 getLinearVarTypes().value().size() != getLinearVars().size())
3889 return emitError() << "Ill-formed type attributes for linear variables";
3890
3891 if (failed(Result: verifyLinearSyms(op: *this)) || failed(Result: verifyPrivateVarList(op&: *this)))
3892 return failure();
3893
3894 return verifyReductionVarList(op: *this, reductionSyms: getReductionSyms(), reductionVars: getReductionVars(),
3895 reductionByref: getReductionByref(), reductionMods: getReductionModsAttr());
3896}
3897
3898LogicalResult WsloopOp::verifyRegions() {
3899 bool isCompositeChildLeaf =
3900 llvm::dyn_cast_if_present<LoopWrapperInterface>(Val: (*this)->getParentOp());
3901
3902 if (LoopWrapperInterface nested = getNestedWrapper()) {
3903 if (!isComposite())
3904 return emitError()
3905 << "'omp.composite' attribute missing from composite wrapper";
3906
3907 // Check for the allowed leaf constructs that may appear in a composite
3908 // construct directly after DO/FOR.
3909 if (!isa<SimdOp>(Val: nested))
3910 return emitError() << "only supported nested wrapper is 'omp.simd'";
3911
3912 } else if (isComposite() && !isCompositeChildLeaf) {
3913 return emitError()
3914 << "'omp.composite' attribute present in non-composite wrapper";
3915 } else if (!isComposite() && isCompositeChildLeaf) {
3916 return emitError()
3917 << "'omp.composite' attribute missing from composite wrapper";
3918 }
3919
3920 return success();
3921}
3922
3923//===----------------------------------------------------------------------===//
3924// Simd construct [2.9.3.1]
3925//===----------------------------------------------------------------------===//
3926
3927void SimdOp::build(OpBuilder &builder, OperationState &state,
3928 const SimdOperands &clauses) {
3929 MLIRContext *ctx = builder.getContext();
3930 SimdOp::build(
3931 odsBuilder&: builder, odsState&: state, aligned_vars: clauses.alignedVars,
3932 alignments: makeArrayAttr(context: ctx, attrs: clauses.alignments), if_expr: clauses.ifExpr,
3933 linear_vars: clauses.linearVars, linear_step_vars: clauses.linearStepVars, linear_var_types: clauses.linearVarTypes,
3934 linear_modifiers: clauses.linearModifiers, linear_syms: makeArrayAttr(context: ctx, attrs: clauses.linearSyms),
3935 nontemporal_vars: clauses.nontemporalVars, order: clauses.order, order_mod: clauses.orderMod,
3936 private_vars: clauses.privateVars, private_syms: makeArrayAttr(context: ctx, attrs: clauses.privateSyms),
3937 private_needs_barrier: clauses.privateNeedsBarrier, reduction_mods: makeArrayAttr(context: ctx, attrs: clauses.reductionMods),
3938 reduction_vars: clauses.reductionVars,
3939 reduction_byref: makeDenseBoolArrayAttr(ctx, boolArray: clauses.reductionByref),
3940 reduction_syms: makeArrayAttr(context: ctx, attrs: clauses.reductionSyms), safelen: clauses.safelen,
3941 simdlen: clauses.simdlen);
3942}
3943
3944LogicalResult SimdOp::verify() {
3945 if (failed(Result: verifyReductionModifiers(op: *this, reductionVars: getReductionVars(),
3946 reductionMods: getReductionModsAttr())))
3947 return failure();
3948
3949 if (getSimdlen().has_value() && getSafelen().has_value() &&
3950 getSimdlen().value() > getSafelen().value())
3951 return emitOpError()
3952 << "simdlen clause and safelen clause are both present, but the "
3953 "simdlen value is not less than or equal to safelen value";
3954
3955 if (verifyAlignedClause(op: *this, alignments: getAlignments(), alignedVars: getAlignedVars()).failed())
3956 return failure();
3957
3958 if (verifyNontemporalClause(op: *this, nontemporalVars: getNontemporalVars()).failed())
3959 return failure();
3960
3961 if (failed(
3962 Result: verifyLinearModifiers(op: *this, linearModifiers: getLinearModifiers(), linearVars: getLinearVars())))
3963 return failure();
3964
3965 bool isCompositeChildLeaf =
3966 llvm::dyn_cast_if_present<LoopWrapperInterface>(Val: (*this)->getParentOp());
3967
3968 if (!isComposite() && isCompositeChildLeaf)
3969 return emitError()
3970 << "'omp.composite' attribute missing from composite wrapper";
3971
3972 if (isComposite() && !isCompositeChildLeaf)
3973 return emitError()
3974 << "'omp.composite' attribute present in non-composite wrapper";
3975
3976 // Firstprivate is not allowed for SIMD in the standard. Check that none of
3977 // the private decls are for firstprivate.
3978 std::optional<ArrayAttr> privateSyms = getPrivateSyms();
3979 if (privateSyms) {
3980 for (const Attribute &sym : *privateSyms) {
3981 auto symRef = cast<SymbolRefAttr>(Val: sym);
3982 omp::PrivateClauseOp privatizer =
3983 SymbolTable::lookupNearestSymbolFrom<omp::PrivateClauseOp>(
3984 from: getOperation(), symbol: symRef);
3985 if (!privatizer)
3986 return emitError() << "Cannot find privatizer '" << symRef << "'";
3987 if (privatizer.getDataSharingType() ==
3988 DataSharingClauseType::FirstPrivate)
3989 return emitError() << "FIRSTPRIVATE cannot be used with SIMD";
3990 }
3991 }
3992
3993 if (failed(Result: verifyPrivateVarList(op&: *this)))
3994 return failure();
3995
3996 if (getLinearVars().size() &&
3997 getLinearVarTypes().value().size() != getLinearVars().size())
3998 return emitError() << "Ill-formed type attributes for linear variables";
3999
4000 if (failed(Result: verifyLinearSyms(op: *this)))
4001 return failure();
4002
4003 llvm::DenseSet<Value> privateVars(llvm::from_range, getPrivateVars());
4004 llvm::DenseSet<Value> reductionVars(llvm::from_range, getReductionVars());
4005 // TODO Check lastprivate vars when their support is added to SimdOp.
4006 for (Value var : getLinearVars()) {
4007 if (privateVars.contains(V: var) || reductionVars.contains(V: var))
4008 return emitOpError()
4009 << "linear variables cannot appear in other data-sharing clauses";
4010 }
4011
4012 return success();
4013}
4014
4015LogicalResult SimdOp::verifyRegions() {
4016 if (getNestedWrapper())
4017 return emitOpError() << "must wrap an 'omp.loop_nest' directly";
4018
4019 return success();
4020}
4021
4022//===----------------------------------------------------------------------===//
4023// Distribute construct [2.9.4.1]
4024//===----------------------------------------------------------------------===//
4025
4026void DistributeOp::build(OpBuilder &builder, OperationState &state,
4027 const DistributeOperands &clauses) {
4028 DistributeOp::build(
4029 odsBuilder&: builder, odsState&: state, allocate_vars: clauses.allocateVars, allocator_vars: clauses.allocatorVars,
4030 allocate_alignments: makeDenseI64ArrayAttr(ctx: builder.getContext(), intArray: clauses.allocateAlignments),
4031 allocate_private_indices: makeDenseI64ArrayAttr(ctx: builder.getContext(),
4032 intArray: clauses.allocatePrivateIndices),
4033 dist_schedule_static: clauses.distScheduleStatic, dist_schedule_chunk_size: clauses.distScheduleChunkSize, order: clauses.order,
4034 order_mod: clauses.orderMod, private_vars: clauses.privateVars,
4035 private_syms: makeArrayAttr(context: builder.getContext(), attrs: clauses.privateSyms),
4036 private_needs_barrier: clauses.privateNeedsBarrier);
4037}
4038
4039LogicalResult DistributeOp::verify() {
4040 if (this->getDistScheduleChunkSize() && !this->getDistScheduleStatic())
4041 return emitOpError() << "chunk size set without "
4042 "dist_schedule_static being present";
4043
4044 if (failed(Result: verifyAllocateClause(
4045 op: getOperation(), allocateVars: getAllocateVars(), allocatorVars: getAllocatorVars(),
4046 allocateAlignments: getAllocateAlignmentsAttr(), allocatePrivateIndices: getAllocatePrivateIndicesAttr(),
4047 privateVars: getPrivateVars(), privateSyms: getPrivateSymsAttr())))
4048 return failure();
4049
4050 if (failed(Result: verifyPrivateVarList(op&: *this)))
4051 return failure();
4052
4053 return success();
4054}
4055
4056LogicalResult DistributeOp::verifyRegions() {
4057 if (LoopWrapperInterface nested = getNestedWrapper()) {
4058 if (!isComposite())
4059 return emitError()
4060 << "'omp.composite' attribute missing from composite wrapper";
4061 // Check for the allowed leaf constructs that may appear in a composite
4062 // construct directly after DISTRIBUTE.
4063 if (isa<WsloopOp>(Val: nested)) {
4064 Operation *parentOp = (*this)->getParentOp();
4065 if (!llvm::dyn_cast_if_present<ParallelOp>(Val: parentOp) ||
4066 !cast<ComposableOpInterface>(Val: parentOp).isComposite()) {
4067 return emitError() << "an 'omp.wsloop' nested wrapper is only allowed "
4068 "when a composite 'omp.parallel' is the direct "
4069 "parent";
4070 }
4071 } else if (!isa<SimdOp>(Val: nested))
4072 return emitError() << "only supported nested wrappers are 'omp.simd' and "
4073 "'omp.wsloop'";
4074 } else if (isComposite()) {
4075 return emitError()
4076 << "'omp.composite' attribute present in non-composite wrapper";
4077 }
4078
4079 return success();
4080}
4081
4082//===----------------------------------------------------------------------===//
4083// DeclareMapperOp / DeclareMapperInfoOp
4084//===----------------------------------------------------------------------===//
4085
4086void DeclareMapperInfoOp::build(OpBuilder &builder, OperationState &state,
4087 const DeclareMapperInfoOperands &clauses) {
4088 DeclareMapperInfoOp::build(odsBuilder&: builder, odsState&: state, map_vars: clauses.mapVars,
4089 map_iterated: clauses.mapIterated);
4090}
4091
4092LogicalResult DeclareMapperInfoOp::verify() {
4093 return verifyMapClause(op: *this, mapVars: getMapVars(), mapIterated: getMapIterated());
4094}
4095
4096LogicalResult DeclareMapperOp::verifyRegions() {
4097 if (!llvm::isa_and_present<DeclareMapperInfoOp>(
4098 Val: getRegion().getBlocks().front().getTerminator()))
4099 return emitOpError() << "expected terminator to be a DeclareMapperInfoOp";
4100
4101 return success();
4102}
4103
4104//===----------------------------------------------------------------------===//
4105// DeclareReductionOp
4106//===----------------------------------------------------------------------===//
4107
4108LogicalResult DeclareReductionOp::verifyRegions() {
4109 if (!getAllocRegion().empty()) {
4110 for (YieldOp yieldOp : getAllocRegion().getOps<YieldOp>()) {
4111 if (yieldOp.getResults().size() != 1 ||
4112 yieldOp.getResults().getTypes()[0] != getType())
4113 return emitOpError() << "expects alloc region to yield a value "
4114 "of the reduction type";
4115 }
4116 }
4117
4118 if (getInitializerRegion().empty())
4119 return emitOpError() << "expects non-empty initializer region";
4120 Block &initializerEntryBlock = getInitializerRegion().front();
4121
4122 if (initializerEntryBlock.getNumArguments() == 1) {
4123 if (!getAllocRegion().empty())
4124 return emitOpError() << "expects two arguments to the initializer region "
4125 "when an allocation region is used";
4126 } else if (initializerEntryBlock.getNumArguments() == 2) {
4127 if (getAllocRegion().empty())
4128 return emitOpError() << "expects one argument to the initializer region "
4129 "when no allocation region is used";
4130 } else {
4131 return emitOpError()
4132 << "expects one or two arguments to the initializer region";
4133 }
4134
4135 for (mlir::Value arg : initializerEntryBlock.getArguments())
4136 if (arg.getType() != getType())
4137 return emitOpError() << "expects initializer region argument to match "
4138 "the reduction type";
4139
4140 for (YieldOp yieldOp : getInitializerRegion().getOps<YieldOp>()) {
4141 if (yieldOp.getResults().size() != 1 ||
4142 yieldOp.getResults().getTypes()[0] != getType())
4143 return emitOpError() << "expects initializer region to yield a value "
4144 "of the reduction type";
4145 }
4146
4147 if (getReductionRegion().empty())
4148 return emitOpError() << "expects non-empty reduction region";
4149 Block &reductionEntryBlock = getReductionRegion().front();
4150 if (reductionEntryBlock.getNumArguments() != 2 ||
4151 reductionEntryBlock.getArgumentTypes()[0] !=
4152 reductionEntryBlock.getArgumentTypes()[1] ||
4153 reductionEntryBlock.getArgumentTypes()[0] != getType())
4154 return emitOpError() << "expects reduction region with two arguments of "
4155 "the reduction type";
4156 for (YieldOp yieldOp : getReductionRegion().getOps<YieldOp>()) {
4157 if (yieldOp.getResults().size() != 1 ||
4158 yieldOp.getResults().getTypes()[0] != getType())
4159 return emitOpError() << "expects reduction region to yield a value "
4160 "of the reduction type";
4161 }
4162
4163 if (!getAtomicReductionRegion().empty()) {
4164 Block &atomicReductionEntryBlock = getAtomicReductionRegion().front();
4165 if (atomicReductionEntryBlock.getNumArguments() != 2 ||
4166 atomicReductionEntryBlock.getArgumentTypes()[0] !=
4167 atomicReductionEntryBlock.getArgumentTypes()[1])
4168 return emitOpError() << "expects atomic reduction region with two "
4169 "arguments of the same type";
4170 auto ptrType = llvm::dyn_cast<PointerLikeType>(
4171 Val: atomicReductionEntryBlock.getArgumentTypes()[0]);
4172 if (!ptrType ||
4173 (ptrType.getElementType() && ptrType.getElementType() != getType()))
4174 return emitOpError() << "expects atomic reduction region arguments to "
4175 "be accumulators containing the reduction type";
4176 }
4177
4178 if (getCleanupRegion().empty())
4179 return success();
4180 Block &cleanupEntryBlock = getCleanupRegion().front();
4181 if (cleanupEntryBlock.getNumArguments() != 1 ||
4182 cleanupEntryBlock.getArgument(i: 0).getType() != getType())
4183 return emitOpError() << "expects cleanup region with one argument "
4184 "of the reduction type";
4185
4186 return success();
4187}
4188
4189//===----------------------------------------------------------------------===//
4190// TaskOp
4191//===----------------------------------------------------------------------===//
4192
4193void TaskOp::build(OpBuilder &builder, OperationState &state,
4194 const TaskOperands &clauses) {
4195 MLIRContext *ctx = builder.getContext();
4196 TaskOp::build(odsBuilder&: builder, odsState&: state, iterated: clauses.iterated, affinity_vars: clauses.affinityVars,
4197 allocate_vars: clauses.allocateVars, allocator_vars: clauses.allocatorVars,
4198 allocate_alignments: makeDenseI64ArrayAttr(ctx, intArray: clauses.allocateAlignments),
4199 allocate_private_indices: makeDenseI64ArrayAttr(ctx, intArray: clauses.allocatePrivateIndices),
4200 depend_kinds: makeArrayAttr(context: ctx, attrs: clauses.dependKinds), depend_vars: clauses.dependVars,
4201 depend_iterated_kinds: makeArrayAttr(context: ctx, attrs: clauses.dependIteratedKinds),
4202 depend_iterated: clauses.dependIterated, final: clauses.final, if_expr: clauses.ifExpr,
4203 in_reduction_vars: clauses.inReductionVars,
4204 in_reduction_byref: makeDenseBoolArrayAttr(ctx, boolArray: clauses.inReductionByref),
4205 in_reduction_syms: makeArrayAttr(context: ctx, attrs: clauses.inReductionSyms), mergeable: clauses.mergeable,
4206 priority: clauses.priority, /*private_vars=*/clauses.privateVars,
4207 /*private_syms=*/makeArrayAttr(context: ctx, attrs: clauses.privateSyms),
4208 private_needs_barrier: clauses.privateNeedsBarrier, threadset: clauses.threadset, untied: clauses.untied,
4209 event_handle: clauses.eventHandle);
4210}
4211
4212LogicalResult TaskOp::verify() {
4213 if (failed(Result: verifyAllocateClause(
4214 op: getOperation(), allocateVars: getAllocateVars(), allocatorVars: getAllocatorVars(),
4215 allocateAlignments: getAllocateAlignmentsAttr(), allocatePrivateIndices: getAllocatePrivateIndicesAttr(),
4216 privateVars: getPrivateVars(), privateSyms: getPrivateSymsAttr())))
4217 return failure();
4218
4219 LogicalResult verifyDependVars =
4220 verifyDependVarList(op: *this, dependKinds: getDependKinds(), dependVars: getDependVars(),
4221 iteratedKinds: getDependIteratedKinds(), iteratedVars: getDependIterated());
4222 if (failed(Result: verifyDependVars))
4223 return verifyDependVars;
4224
4225 if (failed(Result: verifyPrivateVarList(op&: *this)))
4226 return failure();
4227
4228 return verifyReductionVarList(op: *this, reductionSyms: getInReductionSyms(),
4229 reductionVars: getInReductionVars(), reductionByref: getInReductionByref());
4230}
4231
4232//===----------------------------------------------------------------------===//
4233// TaskgroupOp
4234//===----------------------------------------------------------------------===//
4235
4236void TaskgroupOp::build(OpBuilder &builder, OperationState &state,
4237 const TaskgroupOperands &clauses) {
4238 MLIRContext *ctx = builder.getContext();
4239 TaskgroupOp::build(odsBuilder&: builder, odsState&: state, allocate_vars: clauses.allocateVars,
4240 allocator_vars: clauses.allocatorVars,
4241 allocate_alignments: makeDenseI64ArrayAttr(ctx, intArray: clauses.allocateAlignments),
4242 allocate_private_indices: makeDenseI64ArrayAttr(ctx, intArray: clauses.allocatePrivateIndices),
4243 task_reduction_vars: clauses.taskReductionVars,
4244 task_reduction_byref: makeDenseBoolArrayAttr(ctx, boolArray: clauses.taskReductionByref),
4245 task_reduction_syms: makeArrayAttr(context: ctx, attrs: clauses.taskReductionSyms),
4246 /*task_reduction_assume_bounded_recipe_effects=*/nullptr);
4247}
4248
4249LogicalResult TaskgroupOp::verify() {
4250 if (failed(Result: verifyAllocateClause(
4251 op: getOperation(), allocateVars: getAllocateVars(), allocatorVars: getAllocatorVars(),
4252 allocateAlignments: getAllocateAlignmentsAttr(), allocatePrivateIndices: getAllocatePrivateIndicesAttr())))
4253 return failure();
4254
4255 return verifyReductionVarList(op: *this, reductionSyms: getTaskReductionSyms(),
4256 reductionVars: getTaskReductionVars(),
4257 reductionByref: getTaskReductionByref());
4258}
4259
4260static void getTaskReductionEffects(
4261 Operation *op, SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
4262 auto iface = dyn_cast<BlockArgOpenMPOpInterface>(Val: op);
4263 auto taskVars = iface.getTaskReductionVars();
4264 for (auto &source : MutableArrayRef(taskVars.getBase(), taskVars.size())) {
4265 effects.emplace_back(Args: MemoryEffects::Read::get(), Args: &source);
4266 effects.emplace_back(Args: MemoryEffects::Write::get(), Args: &source);
4267 }
4268
4269 for (BlockArgument arg : iface.getTaskReductionBlockArgs()) {
4270 effects.emplace_back(Args: MemoryEffects::Read::get(), Args&: arg);
4271 effects.emplace_back(Args: MemoryEffects::Write::get(), Args&: arg);
4272 }
4273
4274 // The task_reduction clause does not necessarily allocate or free private
4275 // accumulators. We avoid attaching an allocate effect to these accumulators
4276 // as this would incorrectly imply they are fresh storage.
4277 if (!taskVars.empty()) {
4278 effects.emplace_back(Args: MemoryEffects::Allocate::get());
4279 effects.emplace_back(Args: MemoryEffects::Free::get());
4280 }
4281}
4282
4283static void getTaskSchedulingPointEffects(
4284 llvm::SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
4285 &effects) {
4286 effects.emplace_back(Args: MemoryEffects::Read::get());
4287 effects.emplace_back(Args: MemoryEffects::Write::get());
4288}
4289
4290void TaskgroupOp::getEffects(
4291 llvm::SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
4292 &effects) {
4293 getTaskSchedulingPointEffects(effects);
4294
4295 if (!getTaskReductionVars().empty() &&
4296 !getTaskReductionAssumeBoundedRecipeEffects())
4297 addUnknownMemoryEffects(effects);
4298 else
4299 getTaskReductionEffects(op: *this, effects);
4300}
4301
4302//===----------------------------------------------------------------------===//
4303// TaskloopContextOp
4304//===----------------------------------------------------------------------===//
4305
4306void TaskloopContextOp::build(OpBuilder &builder, OperationState &state,
4307 const TaskloopContextOperands &clauses) {
4308 MLIRContext *ctx = builder.getContext();
4309 TaskloopContextOp::build(
4310 odsBuilder&: builder, odsState&: state, allocate_vars: clauses.allocateVars, allocator_vars: clauses.allocatorVars,
4311 allocate_alignments: makeDenseI64ArrayAttr(ctx, intArray: clauses.allocateAlignments),
4312 allocate_private_indices: makeDenseI64ArrayAttr(ctx, intArray: clauses.allocatePrivateIndices), final: clauses.final,
4313 grainsize_mod: clauses.grainsizeMod, grainsize: clauses.grainsize, if_expr: clauses.ifExpr,
4314 in_reduction_vars: clauses.inReductionVars,
4315 in_reduction_byref: makeDenseBoolArrayAttr(ctx, boolArray: clauses.inReductionByref),
4316 in_reduction_syms: makeArrayAttr(context: ctx, attrs: clauses.inReductionSyms), mergeable: clauses.mergeable,
4317 nogroup: clauses.nogroup, num_tasks_mod: clauses.numTasksMod, num_tasks: clauses.numTasks, priority: clauses.priority,
4318 /*private_vars=*/clauses.privateVars,
4319 /*private_syms=*/makeArrayAttr(context: ctx, attrs: clauses.privateSyms),
4320 private_needs_barrier: clauses.privateNeedsBarrier, reduction_mods: makeArrayAttr(context: ctx, attrs: clauses.reductionMods),
4321 reduction_vars: clauses.reductionVars,
4322 reduction_byref: makeDenseBoolArrayAttr(ctx, boolArray: clauses.reductionByref),
4323 reduction_syms: makeArrayAttr(context: ctx, attrs: clauses.reductionSyms), threadset: clauses.threadset,
4324 untied: clauses.untied);
4325 state.addAttribute(name: "omp.combined", attr: UnitAttr::get(context: ctx));
4326}
4327
4328TaskloopWrapperOp TaskloopContextOp::getLoopOp() {
4329 return cast<TaskloopWrapperOp>(
4330 Val&: *llvm::find_if(Range&: getRegion().front(), P: [](mlir::Operation &op) {
4331 return isa<TaskloopWrapperOp>(Val: op);
4332 }));
4333}
4334
4335LogicalResult TaskloopContextOp::verify() {
4336 if (failed(Result: verifyPrivateVarList(op&: *this)))
4337 return failure();
4338 if (failed(Result: verifyAllocateClause(
4339 op: getOperation(), allocateVars: getAllocateVars(), allocatorVars: getAllocatorVars(),
4340 allocateAlignments: getAllocateAlignmentsAttr(), allocatePrivateIndices: getAllocatePrivateIndicesAttr(),
4341 privateVars: getPrivateVars(), privateSyms: getPrivateSymsAttr())))
4342 return failure();
4343
4344 if (failed(Result: verifyReductionVarList(op: *this, reductionSyms: getReductionSyms(),
4345 reductionVars: getReductionVars(), reductionByref: getReductionByref(),
4346 reductionMods: getReductionModsAttr())) ||
4347 failed(Result: verifyReductionVarList(op: *this, reductionSyms: getInReductionSyms(),
4348 reductionVars: getInReductionVars(),
4349 reductionByref: getInReductionByref())))
4350 return failure();
4351
4352 if (!getReductionVars().empty() && getNogroup())
4353 return emitError(message: "if a reduction clause is present on the taskloop "
4354 "directive, the nogroup clause must not be specified");
4355 for (auto var : getReductionVars()) {
4356 if (llvm::is_contained(Range: getInReductionVars(), Element: var))
4357 return emitError(message: "the same list item cannot appear in both a reduction "
4358 "and an in_reduction clause");
4359 }
4360
4361 if (getGrainsize() && getNumTasks()) {
4362 return emitError(
4363 message: "the grainsize clause and num_tasks clause are mutually exclusive and "
4364 "may not appear on the same taskloop directive");
4365 }
4366
4367 // Without this restriction, any compound construct including `taskloop` would
4368 // fail to correctly identify the whole chain of operations (see
4369 // ComposableOpInterface::findCapturedOp()), as well as failing to do so even
4370 // for standalone `taskloop` constructs.
4371 if (!isCombined())
4372 return emitOpError(message: "must always contain the 'omp.combined' attribute");
4373
4374 return success();
4375}
4376
4377LogicalResult TaskloopContextOp::verifyRegions() {
4378 Region &region = getRegion();
4379 auto loopWrapperIt = llvm::find_if(Range&: region.front(), P: [](mlir::Operation &op) {
4380 return isa<TaskloopWrapperOp>(Val: op);
4381 });
4382 if (loopWrapperIt == region.front().end())
4383 return emitOpError()
4384 << "expected a TaskloopWrapperOp directly nested in the region";
4385
4386 auto loopWrapperOp = cast<TaskloopWrapperOp>(Val&: *loopWrapperIt);
4387 auto loopNestOp = dyn_cast<LoopNestOp>(Val: loopWrapperOp.getWrappedLoop());
4388 // This will fail the verifier for TaskloopWrapperOp and print an error
4389 // message there.
4390 if (!loopNestOp)
4391 return failure();
4392
4393 std::function<bool(Value)> isValidBoundValue = [&](Value value) -> bool {
4394 Region *valueRegion = value.getParentRegion();
4395 // A loop bound value defined outside of the taskloop context region is
4396 // valid. A region is considered an ancestor of itself.
4397 if (!region.isAncestor(other: valueRegion))
4398 return true;
4399
4400 Operation *defOp = value.getDefiningOp();
4401 if (!defOp || defOp->getNumRegions() != 0 || !isPure(op: defOp))
4402 return false;
4403
4404 return llvm::all_of(Range: defOp->getOperands(), P: isValidBoundValue);
4405 };
4406 auto hasUnsupportedTaskloopLocalBound = [&](OperandRange range) -> bool {
4407 return llvm::any_of(Range&: range,
4408 P: [&](Value value) { return !isValidBoundValue(value); });
4409 };
4410
4411 if (hasUnsupportedTaskloopLocalBound(loopNestOp.getLoopLowerBounds()) ||
4412 hasUnsupportedTaskloopLocalBound(loopNestOp.getLoopUpperBounds()) ||
4413 hasUnsupportedTaskloopLocalBound(loopNestOp.getLoopSteps())) {
4414 return emitOpError()
4415 << "expects loop bounds and steps to be defined outside of the "
4416 "taskloop.context region or by pure, regionless operations "
4417 "that do not depend on block arguments";
4418 }
4419
4420 return success();
4421}
4422
4423//===----------------------------------------------------------------------===//
4424// TaskloopWrapperOp
4425//===----------------------------------------------------------------------===//
4426
4427void TaskloopWrapperOp::build(OpBuilder &builder, OperationState &state,
4428 const TaskloopWrapperOperands &clauses) {
4429 TaskloopWrapperOp::build(odsBuilder&: builder, odsState&: state);
4430}
4431
4432TaskloopContextOp TaskloopWrapperOp::getTaskloopContext() {
4433 return dyn_cast<TaskloopContextOp>(Val: getOperation()->getParentOp());
4434}
4435
4436LogicalResult TaskloopWrapperOp::verify() {
4437 TaskloopContextOp context = getTaskloopContext();
4438 if (!context)
4439 return emitOpError() << "expected to be nested in a taskloop context op";
4440 return success();
4441}
4442
4443LogicalResult TaskloopWrapperOp::verifyRegions() {
4444 if (LoopWrapperInterface nested = getNestedWrapper()) {
4445 if (!isComposite())
4446 return emitError()
4447 << "'omp.composite' attribute missing from composite wrapper";
4448
4449 // Check for the allowed leaf constructs that may appear in a composite
4450 // construct directly after TASKLOOP.
4451 if (!isa<SimdOp>(Val: nested))
4452 return emitError() << "only supported nested wrapper is 'omp.simd'";
4453 } else if (isComposite()) {
4454 return emitError()
4455 << "'omp.composite' attribute present in non-composite wrapper";
4456 }
4457
4458 return success();
4459}
4460
4461//===----------------------------------------------------------------------===//
4462// LoopNestOp
4463//===----------------------------------------------------------------------===//
4464
4465ParseResult LoopNestOp::parse(OpAsmParser &parser, OperationState &result) {
4466 // Parse an opening `(` followed by induction variables followed by `)`
4467 SmallVector<OpAsmParser::Argument> ivs;
4468 SmallVector<OpAsmParser::UnresolvedOperand> lbs, ubs;
4469 Type loopVarType;
4470 if (parser.parseArgumentList(result&: ivs, delimiter: OpAsmParser::Delimiter::Paren) ||
4471 parser.parseColonType(result&: loopVarType) ||
4472 // Parse loop bounds.
4473 parser.parseEqual() ||
4474 parser.parseOperandList(result&: lbs, requiredOperandCount: ivs.size(), delimiter: OpAsmParser::Delimiter::Paren) ||
4475 parser.parseKeyword(keyword: "to") ||
4476 parser.parseOperandList(result&: ubs, requiredOperandCount: ivs.size(), delimiter: OpAsmParser::Delimiter::Paren))
4477 return failure();
4478
4479 for (auto &iv : ivs)
4480 iv.type = loopVarType;
4481
4482 auto *ctx = parser.getBuilder().getContext();
4483 // Parse "inclusive" flag.
4484 if (succeeded(Result: parser.parseOptionalKeyword(keyword: "inclusive")))
4485 result.addAttribute(name: "loop_inclusive", attr: UnitAttr::get(context: ctx));
4486
4487 // Parse step values.
4488 SmallVector<OpAsmParser::UnresolvedOperand> steps;
4489 if (parser.parseKeyword(keyword: "step") ||
4490 parser.parseOperandList(result&: steps, requiredOperandCount: ivs.size(), delimiter: OpAsmParser::Delimiter::Paren))
4491 return failure();
4492
4493 // Parse collapse
4494 int64_t value = 0;
4495 if (!parser.parseOptionalKeyword(keyword: "collapse") &&
4496 (parser.parseLParen() || parser.parseInteger(result&: value) ||
4497 parser.parseRParen()))
4498 return failure();
4499 if (value > 1)
4500 result.addAttribute(
4501 name: "collapse_num_loops",
4502 attr: IntegerAttr::get(type: parser.getBuilder().getI64Type(), value));
4503
4504 // Parse tiles
4505 SmallVector<int64_t> tiles;
4506 auto parseTiles = [&]() -> ParseResult {
4507 int64_t tile;
4508 if (parser.parseInteger(result&: tile))
4509 return failure();
4510 tiles.push_back(Elt: tile);
4511 return success();
4512 };
4513
4514 if (!parser.parseOptionalKeyword(keyword: "tiles") &&
4515 (parser.parseLParen() || parser.parseCommaSeparatedList(parseElementFn: parseTiles) ||
4516 parser.parseRParen()))
4517 return failure();
4518
4519 if (tiles.size() > 0)
4520 result.addAttribute(name: "tile_sizes", attr: DenseI64ArrayAttr::get(context: ctx, content: tiles));
4521
4522 // Parse the body.
4523 Region *region = result.addRegion();
4524 if (parser.parseRegion(region&: *region, arguments: ivs))
4525 return failure();
4526
4527 // Resolve operands.
4528 if (parser.resolveOperands(operands&: lbs, type: loopVarType, result&: result.operands) ||
4529 parser.resolveOperands(operands&: ubs, type: loopVarType, result&: result.operands) ||
4530 parser.resolveOperands(operands&: steps, type: loopVarType, result&: result.operands))
4531 return failure();
4532
4533 // Parse the optional attribute list.
4534 return parser.parseOptionalAttrDict(result&: result.attributes);
4535}
4536
4537void LoopNestOp::print(OpAsmPrinter &p) {
4538 Region &region = getRegion();
4539 auto args = region.getArguments();
4540 p << " (" << args << ") : " << args[0].getType() << " = ("
4541 << getLoopLowerBounds() << ") to (" << getLoopUpperBounds() << ") ";
4542 if (getLoopInclusive())
4543 p << "inclusive ";
4544 p << "step (" << getLoopSteps() << ") ";
4545 if (int64_t numCollapse = getCollapseNumLoops())
4546 if (numCollapse > 1)
4547 p << "collapse(" << numCollapse << ") ";
4548
4549 if (const auto tiles = getTileSizes())
4550 p << "tiles(" << tiles.value() << ") ";
4551
4552 p.printRegion(blocks&: region, /*printEntryBlockArgs=*/false);
4553}
4554
4555void LoopNestOp::build(OpBuilder &builder, OperationState &state,
4556 const LoopNestOperands &clauses) {
4557 MLIRContext *ctx = builder.getContext();
4558 LoopNestOp::build(odsBuilder&: builder, odsState&: state, collapse_num_loops: clauses.collapseNumLoops,
4559 loop_lower_bounds: clauses.loopLowerBounds, loop_upper_bounds: clauses.loopUpperBounds,
4560 loop_steps: clauses.loopSteps, loop_inclusive: clauses.loopInclusive,
4561 tile_sizes: makeDenseI64ArrayAttr(ctx, intArray: clauses.tileSizes));
4562}
4563
4564LogicalResult LoopNestOp::verify() {
4565 if (getLoopLowerBounds().empty())
4566 return emitOpError() << "must represent at least one loop";
4567
4568 if (getLoopLowerBounds().size() != getIVs().size())
4569 return emitOpError() << "number of range arguments and IVs do not match";
4570
4571 for (auto [lb, iv] : llvm::zip_equal(t: getLoopLowerBounds(), u: getIVs())) {
4572 if (lb.getType() != iv.getType())
4573 return emitOpError()
4574 << "range argument type does not match corresponding IV type";
4575 }
4576
4577 uint64_t numIVs = getIVs().size();
4578
4579 if (const auto &numCollapse = getCollapseNumLoops())
4580 if (numCollapse > numIVs)
4581 return emitOpError()
4582 << "collapse value is larger than the number of loops";
4583
4584 if (const auto &tiles = getTileSizes())
4585 if (tiles.value().size() > numIVs)
4586 return emitOpError() << "too few canonical loops for tile dimensions";
4587
4588 if (!llvm::dyn_cast_if_present<LoopWrapperInterface>(Val: (*this)->getParentOp()))
4589 return emitOpError() << "expects parent op to be a loop wrapper";
4590
4591 return success();
4592}
4593
4594void LoopNestOp::gatherWrappers(
4595 SmallVectorImpl<LoopWrapperInterface> &wrappers) {
4596 Operation *parent = (*this)->getParentOp();
4597 while (auto wrapper =
4598 llvm::dyn_cast_if_present<LoopWrapperInterface>(Val: parent)) {
4599 wrappers.push_back(Elt: wrapper);
4600 parent = parent->getParentOp();
4601 }
4602}
4603
4604//===----------------------------------------------------------------------===//
4605// OpenMP canonical loop handling
4606//===----------------------------------------------------------------------===//
4607
4608std::tuple<NewCliOp, OpOperand *, OpOperand *>
4609mlir::omp ::decodeCli(Value cli) {
4610
4611 // Defining a CLI for a generated loop is optional; if there is none then
4612 // there is no followup-tranformation
4613 if (!cli)
4614 return {{}, nullptr, nullptr};
4615
4616 assert(cli.getType() == CanonicalLoopInfoType::get(cli.getContext()) &&
4617 "Unexpected type of cli");
4618
4619 NewCliOp create = cast<NewCliOp>(Val: cli.getDefiningOp());
4620 OpOperand *gen = nullptr;
4621 OpOperand *cons = nullptr;
4622 for (OpOperand &use : cli.getUses()) {
4623 auto op = cast<LoopTransformationInterface>(Val: use.getOwner());
4624
4625 unsigned opnum = use.getOperandNumber();
4626 if (op.isGeneratee(opnum)) {
4627 assert(!gen && "Each CLI may have at most one def");
4628 gen = &use;
4629 } else if (op.isApplyee(opnum)) {
4630 assert(!cons && "Each CLI may have at most one consumer");
4631 cons = &use;
4632 } else {
4633 llvm_unreachable("Unexpected operand for a CLI");
4634 }
4635 }
4636
4637 return {create, gen, cons};
4638}
4639
4640ClauseProcBindKind
4641mlir::omp::convertProcBindKind(llvm::omp::ProcBindKind kind) {
4642 switch (kind) {
4643 case llvm::omp::ProcBindKind::OMP_PROC_BIND_close:
4644 return ClauseProcBindKind::Close;
4645 case llvm::omp::ProcBindKind::OMP_PROC_BIND_master:
4646 return ClauseProcBindKind::Master;
4647 case llvm::omp::ProcBindKind::OMP_PROC_BIND_primary:
4648 return ClauseProcBindKind::Primary;
4649 case llvm::omp::ProcBindKind::OMP_PROC_BIND_spread:
4650 return ClauseProcBindKind::Spread;
4651 case llvm::omp::ProcBindKind::OMP_PROC_BIND_default:
4652 case llvm::omp::ProcBindKind::OMP_PROC_BIND_unknown:
4653 break;
4654 }
4655 llvm_unreachable("unexpected proc-bind kind");
4656}
4657
4658void NewCliOp::build(::mlir::OpBuilder &odsBuilder,
4659 ::mlir::OperationState &odsState) {
4660 odsState.addTypes(newTypes: CanonicalLoopInfoType::get(ctx: odsBuilder.getContext()));
4661}
4662
4663void NewCliOp::getAsmResultNames(OpAsmSetValueNameFn setNameFn) {
4664 Value result = getResult();
4665 auto [newCli, gen, cons] = decodeCli(cli: result);
4666
4667 // Structured binding `gen` cannot be captured in lambdas before C++20
4668 OpOperand *generator = gen;
4669
4670 // Derive the CLI variable name from its generator:
4671 // * "canonloop" for omp.canonical_loop
4672 // * custom name for loop transformation generatees
4673 // * "cli" as fallback if no generator
4674 // * "_r<idx>" suffix for nested loops, where <idx> is the sequential order
4675 // at that level
4676 // * "_s<idx>" suffix for operations with multiple regions, where <idx> is
4677 // the index of that region
4678 std::string cliName{"cli"};
4679 if (gen) {
4680 cliName =
4681 TypeSwitch<Operation *, std::string>(gen->getOwner())
4682 .Case(caseFn: [&](CanonicalLoopOp op) {
4683 return generateLoopNestingName(prefix: "canonloop", op);
4684 })
4685 .Case(caseFn: [&](UnrollHeuristicOp op) -> std::string {
4686 llvm_unreachable("heuristic unrolling does not generate a loop");
4687 })
4688 .Case(caseFn: [&](FuseOp op) -> std::string {
4689 unsigned opnum = generator->getOperandNumber();
4690 // The position of the first loop to be fused is the same position
4691 // as the resulting fused loop
4692 if (op.getFirst().has_value() && opnum != op.getFirst().value())
4693 return "canonloop_fuse";
4694 else
4695 return "fused";
4696 })
4697 .Case(caseFn: [&](TileOp op) -> std::string {
4698 auto [generateesFirst, generateesCount] =
4699 op.getGenerateesODSOperandIndexAndLength();
4700 unsigned firstGrid = generateesFirst;
4701 unsigned firstIntratile = generateesFirst + generateesCount / 2;
4702 unsigned end = generateesFirst + generateesCount;
4703 unsigned opnum = generator->getOperandNumber();
4704 // In the OpenMP apply and looprange clauses, indices are 1-based
4705 if (firstGrid <= opnum && opnum < firstIntratile) {
4706 unsigned gridnum = opnum - firstGrid + 1;
4707 return ("grid" + Twine(gridnum)).str();
4708 }
4709 if (firstIntratile <= opnum && opnum < end) {
4710 unsigned intratilenum = opnum - firstIntratile + 1;
4711 return ("intratile" + Twine(intratilenum)).str();
4712 }
4713 llvm_unreachable("Unexpected generatee argument");
4714 })
4715 .DefaultUnreachable(message: "TODO: Custom name for this operation");
4716 }
4717
4718 setNameFn(result, cliName);
4719}
4720
4721LogicalResult NewCliOp::verify() {
4722 Value cli = getResult();
4723
4724 assert(cli.getType() == CanonicalLoopInfoType::get(cli.getContext()) &&
4725 "Unexpected type of cli");
4726
4727 // Check that the CLI is used in at most generator and one consumer
4728 OpOperand *gen = nullptr;
4729 OpOperand *cons = nullptr;
4730 for (mlir::OpOperand &use : cli.getUses()) {
4731 auto op = dyn_cast<mlir::omp::LoopTransformationInterface>(Val: use.getOwner());
4732 if (!op) {
4733 InFlightDiagnostic error =
4734 emitOpError(message: "CLI must only be used by loop transformations");
4735 error.attachNote(noteLoc: use.getOwner()->getLoc()).append(arg: "invalid use here:");
4736 return error;
4737 }
4738
4739 unsigned opnum = use.getOperandNumber();
4740 if (op.isGeneratee(opnum)) {
4741 if (gen) {
4742 InFlightDiagnostic error =
4743 emitOpError(message: "CLI must have at most one generator");
4744 error.attachNote(noteLoc: gen->getOwner()->getLoc())
4745 .append(arg: "first generator here:");
4746 error.attachNote(noteLoc: use.getOwner()->getLoc())
4747 .append(arg: "second generator here:");
4748 return error;
4749 }
4750
4751 gen = &use;
4752 } else if (op.isApplyee(opnum)) {
4753 if (cons) {
4754 InFlightDiagnostic error =
4755 emitOpError(message: "CLI must have at most one consumer");
4756 error.attachNote(noteLoc: cons->getOwner()->getLoc())
4757 .append(arg: "first consumer here:")
4758 .appendOp(op&: *cons->getOwner(),
4759 flags: OpPrintingFlags().printGenericOpForm());
4760 error.attachNote(noteLoc: use.getOwner()->getLoc())
4761 .append(arg: "second consumer here:")
4762 .appendOp(op&: *use.getOwner(), flags: OpPrintingFlags().printGenericOpForm());
4763 return error;
4764 }
4765
4766 cons = &use;
4767 } else {
4768 llvm_unreachable("Unexpected operand for a CLI");
4769 }
4770 }
4771
4772 // If the CLI is source of a transformation, it must have a generator
4773 if (cons && !gen) {
4774 InFlightDiagnostic error = emitOpError(message: "CLI has no generator");
4775 error.attachNote(noteLoc: cons->getOwner()->getLoc())
4776 .append(arg: "see consumer here: ")
4777 .appendOp(op&: *cons->getOwner(), flags: OpPrintingFlags().printGenericOpForm());
4778 return error;
4779 }
4780
4781 return success();
4782}
4783
4784void CanonicalLoopOp::build(OpBuilder &odsBuilder, OperationState &odsState,
4785 Value tripCount) {
4786 odsState.addOperands(newOperands: tripCount);
4787 odsState.addOperands(newOperands: Value());
4788 (void)odsState.addRegion();
4789}
4790
4791void CanonicalLoopOp::build(OpBuilder &odsBuilder, OperationState &odsState,
4792 Value tripCount, ::mlir::Value cli) {
4793 odsState.addOperands(newOperands: tripCount);
4794 odsState.addOperands(newOperands: cli);
4795 (void)odsState.addRegion();
4796}
4797
4798void CanonicalLoopOp::getAsmBlockNames(OpAsmSetBlockNameFn setNameFn) {
4799 setNameFn(&getRegion().front(), "body_entry");
4800}
4801
4802void CanonicalLoopOp::getAsmBlockArgumentNames(Region &region,
4803 OpAsmSetValueNameFn setNameFn) {
4804 std::string ivName = generateLoopNestingName(prefix: "iv", op: *this);
4805 setNameFn(region.getArgument(i: 0), ivName);
4806}
4807
4808void CanonicalLoopOp::print(OpAsmPrinter &p) {
4809 if (getCli())
4810 p << '(' << getCli() << ')';
4811 p << ' ' << getInductionVar() << " : " << getInductionVar().getType()
4812 << " in range(" << getTripCount() << ") ";
4813
4814 p.printRegion(blocks&: getRegion(), /*printEntryBlockArgs=*/false,
4815 /*printBlockTerminators=*/true);
4816
4817 p.printOptionalAttrDict(attrs: (*this)->getDiscardableAttrDictionary().getValue());
4818}
4819
4820mlir::ParseResult CanonicalLoopOp::parse(::mlir::OpAsmParser &parser,
4821 ::mlir::OperationState &result) {
4822 CanonicalLoopInfoType cliType =
4823 CanonicalLoopInfoType::get(ctx: parser.getContext());
4824
4825 // Parse (optional) omp.cli identifier
4826 OpAsmParser::UnresolvedOperand cli;
4827 SmallVector<mlir::Value, 1> cliOperand;
4828 if (!parser.parseOptionalLParen()) {
4829 if (parser.parseOperand(result&: cli) ||
4830 parser.resolveOperand(operand: cli, type: cliType, result&: cliOperand) || parser.parseRParen())
4831 return failure();
4832 }
4833
4834 // We derive the type of tripCount from inductionVariable. MLIR requires the
4835 // type of tripCount to be known when calling resolveOperand so we have parse
4836 // the type before processing the inductionVariable.
4837 OpAsmParser::Argument inductionVariable;
4838 OpAsmParser::UnresolvedOperand tripcount;
4839 if (parser.parseArgument(result&: inductionVariable, /*allowType*/ true) ||
4840 parser.parseKeyword(keyword: "in") || parser.parseKeyword(keyword: "range") ||
4841 parser.parseLParen() || parser.parseOperand(result&: tripcount) ||
4842 parser.parseRParen() ||
4843 parser.resolveOperand(operand: tripcount, type: inductionVariable.type, result&: result.operands))
4844 return failure();
4845
4846 // Parse the loop body.
4847 Region *region = result.addRegion();
4848 if (parser.parseRegion(region&: *region, arguments: {inductionVariable}))
4849 return failure();
4850
4851 // We parsed the cli operand forst, but because it is optional, it must be
4852 // last in the operand list.
4853 result.operands.append(RHS: cliOperand);
4854
4855 // Parse the optional attribute list.
4856 if (parser.parseOptionalAttrDict(result&: result.attributes))
4857 return failure();
4858
4859 return mlir::success();
4860}
4861
4862LogicalResult CanonicalLoopOp::verify() {
4863 // The region's entry must accept the induction variable
4864 // It can also be empty if just created
4865 if (!getRegion().empty()) {
4866 Region &region = getRegion();
4867 if (region.getNumArguments() != 1)
4868 return emitOpError(
4869 message: "Canonical loop region must have exactly one argument");
4870
4871 if (getInductionVar().getType() != getTripCount().getType())
4872 return emitOpError(
4873 message: "Region argument must be the same type as the trip count");
4874 }
4875
4876 return success();
4877}
4878
4879Value CanonicalLoopOp::getInductionVar() { return getRegion().getArgument(i: 0); }
4880
4881std::pair<unsigned, unsigned>
4882CanonicalLoopOp::getApplyeesODSOperandIndexAndLength() {
4883 // No applyees
4884 return {0, 0};
4885}
4886
4887std::pair<unsigned, unsigned>
4888CanonicalLoopOp::getGenerateesODSOperandIndexAndLength() {
4889 return getODSOperandIndexAndLength(index: odsIndex_cli);
4890}
4891
4892//===----------------------------------------------------------------------===//
4893// UnrollHeuristicOp
4894//===----------------------------------------------------------------------===//
4895
4896void UnrollHeuristicOp::build(::mlir::OpBuilder &odsBuilder,
4897 ::mlir::OperationState &odsState,
4898 ::mlir::Value cli) {
4899 odsState.addOperands(newOperands: cli);
4900}
4901
4902void UnrollHeuristicOp::print(OpAsmPrinter &p) {
4903 p << '(' << getApplyee() << ')';
4904
4905 p.printOptionalAttrDict(attrs: (*this)->getDiscardableAttrDictionary().getValue());
4906}
4907
4908mlir::ParseResult UnrollHeuristicOp::parse(::mlir::OpAsmParser &parser,
4909 ::mlir::OperationState &result) {
4910 auto cliType = CanonicalLoopInfoType::get(ctx: parser.getContext());
4911
4912 if (parser.parseLParen())
4913 return failure();
4914
4915 OpAsmParser::UnresolvedOperand applyee;
4916 if (parser.parseOperand(result&: applyee) ||
4917 parser.resolveOperand(operand: applyee, type: cliType, result&: result.operands))
4918 return failure();
4919
4920 if (parser.parseRParen())
4921 return failure();
4922
4923 // Optional output loop (full unrolling has none)
4924 if (!parser.parseOptionalArrow()) {
4925 if (parser.parseLParen() || parser.parseRParen())
4926 return failure();
4927 }
4928
4929 // Parse the optional attribute list.
4930 if (parser.parseOptionalAttrDict(result&: result.attributes))
4931 return failure();
4932
4933 return mlir::success();
4934}
4935
4936std::pair<unsigned, unsigned>
4937UnrollHeuristicOp ::getApplyeesODSOperandIndexAndLength() {
4938 return getODSOperandIndexAndLength(index: odsIndex_applyee);
4939}
4940
4941std::pair<unsigned, unsigned>
4942UnrollHeuristicOp::getGenerateesODSOperandIndexAndLength() {
4943 return {0, 0};
4944}
4945
4946//===----------------------------------------------------------------------===//
4947// UnrollFullOp
4948//===----------------------------------------------------------------------===//
4949
4950void UnrollFullOp::build(::mlir::OpBuilder &odsBuilder,
4951 ::mlir::OperationState &odsState, ::mlir::Value cli) {
4952 odsState.addOperands(newOperands: cli);
4953}
4954
4955void UnrollFullOp::print(OpAsmPrinter &p) {
4956 p << '(' << getApplyee() << ')';
4957
4958 p.printOptionalAttrDict(attrs: (*this)->getDiscardableAttrDictionary().getValue());
4959}
4960
4961mlir::ParseResult UnrollFullOp::parse(::mlir::OpAsmParser &parser,
4962 ::mlir::OperationState &result) {
4963 auto cliType = CanonicalLoopInfoType::get(ctx: parser.getContext());
4964
4965 if (parser.parseLParen())
4966 return failure();
4967
4968 OpAsmParser::UnresolvedOperand applyee;
4969 if (parser.parseOperand(result&: applyee) ||
4970 parser.resolveOperand(operand: applyee, type: cliType, result&: result.operands))
4971 return failure();
4972
4973 if (parser.parseRParen())
4974 return failure();
4975
4976 // Optional output loop; full unrolling has none.
4977 if (!parser.parseOptionalArrow()) {
4978 if (parser.parseLParen() || parser.parseRParen())
4979 return failure();
4980 }
4981
4982 // Parse the optional attribute list.
4983 if (parser.parseOptionalAttrDict(result&: result.attributes))
4984 return failure();
4985
4986 return mlir::success();
4987}
4988
4989std::pair<unsigned, unsigned>
4990UnrollFullOp::getApplyeesODSOperandIndexAndLength() {
4991 return getODSOperandIndexAndLength(index: odsIndex_applyee);
4992}
4993
4994std::pair<unsigned, unsigned>
4995UnrollFullOp::getGenerateesODSOperandIndexAndLength() {
4996 return {0, 0};
4997}
4998
4999LogicalResult UnrollFullOp::verify() {
5000 auto [create, gen, cons] = decodeCli(cli: getApplyee());
5001 if (!gen)
5002 return emitOpError() << "applyee CLI has no generator";
5003
5004 // Full unrolling leaves no loop, so the trip count must be constant. Only
5005 // omp.canonical_loop states one.
5006 if (auto loop = dyn_cast<CanonicalLoopOp>(Val: gen->getOwner())) {
5007 if (!matchPattern(value: loop.getTripCount(), pattern: m_Constant()))
5008 return emitOpError() << "applyee loop must have a constant trip count";
5009 }
5010
5011 return success();
5012}
5013
5014//===----------------------------------------------------------------------===//
5015// UnrollPartialOp
5016//===----------------------------------------------------------------------===//
5017
5018void UnrollPartialOp::build(::mlir::OpBuilder &odsBuilder,
5019 ::mlir::OperationState &odsState, ::mlir::Value cli,
5020 uint64_t unrollFactor) {
5021 odsState.addOperands(newOperands: cli);
5022 Properties &props = odsState.getOrAddProperties<Properties>();
5023 props.unroll_factor = odsBuilder.getI64IntegerAttr(value: unrollFactor);
5024}
5025
5026void UnrollPartialOp::print(OpAsmPrinter &p) {
5027 p << '(' << getApplyee() << ')';
5028
5029 SmallVector<NamedAttribute> attrs((*this)->getDiscardableAttrs());
5030 attrs.emplace_back(Args: getUnrollFactorAttrName(), Args: getUnrollFactorAttr());
5031 llvm::sort(C&: attrs);
5032 p.printOptionalAttrDict(attrs);
5033}
5034
5035mlir::ParseResult UnrollPartialOp::parse(::mlir::OpAsmParser &parser,
5036 ::mlir::OperationState &result) {
5037 auto cliType = CanonicalLoopInfoType::get(ctx: parser.getContext());
5038
5039 if (parser.parseLParen())
5040 return failure();
5041
5042 OpAsmParser::UnresolvedOperand applyee;
5043 if (parser.parseOperand(result&: applyee) ||
5044 parser.resolveOperand(operand: applyee, type: cliType, result&: result.operands))
5045 return failure();
5046
5047 if (parser.parseRParen())
5048 return failure();
5049
5050 // The unroll factor is carried by the `unroll_factor` attribute.
5051 if (parser.parseOptionalAttrDict(result&: result.attributes))
5052 return failure();
5053
5054 return mlir::success();
5055}
5056
5057std::pair<unsigned, unsigned>
5058UnrollPartialOp::getApplyeesODSOperandIndexAndLength() {
5059 return getODSOperandIndexAndLength(index: odsIndex_applyee);
5060}
5061
5062std::pair<unsigned, unsigned>
5063UnrollPartialOp::getGenerateesODSOperandIndexAndLength() {
5064 return {0, 0};
5065}
5066
5067//===----------------------------------------------------------------------===//
5068// TileOp
5069//===----------------------------------------------------------------------===//
5070
5071static void printLoopTransformClis(OpAsmPrinter &p, TileOp op,
5072 OperandRange generatees,
5073 OperandRange applyees) {
5074 if (!generatees.empty())
5075 p << '(' << llvm::interleaved(R: generatees) << ')';
5076
5077 if (!applyees.empty())
5078 p << " <- (" << llvm::interleaved(R: applyees) << ')';
5079}
5080
5081static ParseResult parseLoopTransformClis(
5082 OpAsmParser &parser,
5083 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &generateesOperands,
5084 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &applyeesOperands) {
5085 if (parser.parseOptionalLess()) {
5086 // Syntax 1: generatees present
5087
5088 if (parser.parseOperandList(result&: generateesOperands,
5089 delimiter: mlir::OpAsmParser::Delimiter::Paren))
5090 return failure();
5091
5092 if (parser.parseLess())
5093 return failure();
5094 } else {
5095 // Syntax 2: generatees omitted
5096 }
5097
5098 // Parse `<-` (`<` has already been parsed)
5099 if (parser.parseMinus())
5100 return failure();
5101
5102 if (parser.parseOperandList(result&: applyeesOperands,
5103 delimiter: mlir::OpAsmParser::Delimiter::Paren))
5104 return failure();
5105
5106 return success();
5107}
5108
5109/// Check properties of the loop nest consisting of the transformation's
5110/// applyees:
5111/// 1. They are nested inside each other
5112/// 2. They are perfectly nested
5113/// (no code with side-effects in-between the loops)
5114/// 3. They are rectangular
5115/// (loop bounds are invariant in respect to the outer loops)
5116///
5117/// TODO: Generalize for LoopTransformationInterface.
5118static LogicalResult checkApplyeesNesting(TileOp op) {
5119 // Collect the loops from the nest
5120 bool isOnlyCanonLoops = true;
5121 SmallVector<CanonicalLoopOp> canonLoops;
5122 for (Value applyee : op.getApplyees()) {
5123 auto [create, gen, cons] = decodeCli(cli: applyee);
5124
5125 if (!gen)
5126 return op.emitOpError() << "applyee CLI has no generator";
5127
5128 auto loop = dyn_cast_or_null<CanonicalLoopOp>(Val: gen->getOwner());
5129 canonLoops.push_back(Elt: loop);
5130 if (!loop)
5131 isOnlyCanonLoops = false;
5132 }
5133
5134 // FIXME: We currently can only verify non-rectangularity and perfect nest of
5135 // omp.canonical_loop.
5136 if (!isOnlyCanonLoops)
5137 return success();
5138
5139 DenseSet<Value> parentIVs;
5140 for (auto i : llvm::seq<int>(Begin: 1, End: canonLoops.size())) {
5141 auto parentLoop = canonLoops[i - 1];
5142 auto loop = canonLoops[i];
5143
5144 if (parentLoop.getOperation() != loop.getOperation()->getParentOp())
5145 return op.emitOpError()
5146 << "tiled loop nest must be nested within each other";
5147
5148 parentIVs.insert(V: parentLoop.getInductionVar());
5149
5150 // Canonical loop must be perfectly nested, i.e. the body of the parent must
5151 // only contain the omp.canonical_loop of the nested loops, and
5152 // omp.terminator
5153 bool isPerfectlyNested = [&]() {
5154 auto &parentBody = parentLoop.getRegion();
5155 if (!parentBody.hasOneBlock())
5156 return false;
5157 auto &parentBlock = parentBody.getBlocks().front();
5158
5159 auto nestedLoopIt = parentBlock.begin();
5160 if (nestedLoopIt == parentBlock.end() ||
5161 (&*nestedLoopIt != loop.getOperation()))
5162 return false;
5163
5164 auto termIt = std::next(x: nestedLoopIt);
5165 if (termIt == parentBlock.end() || !isa<TerminatorOp>(Val: termIt))
5166 return false;
5167
5168 if (std::next(x: termIt) != parentBlock.end())
5169 return false;
5170
5171 return true;
5172 }();
5173 if (!isPerfectlyNested)
5174 return op.emitOpError() << "tiled loop nest must be perfectly nested";
5175
5176 if (parentIVs.contains(V: loop.getTripCount()))
5177 return op.emitOpError() << "tiled loop nest must be rectangular";
5178 }
5179
5180 // TODO: The tile sizes must be computed before the loop, but checking this
5181 // requires dominance analysis. For instance:
5182 //
5183 // %canonloop = omp.new_cli
5184 // omp.canonical_loop(%canonloop) %iv : i32 in range(%tc) {
5185 // // write to %x
5186 // omp.terminator
5187 // }
5188 // %ts = llvm.load %x
5189 // omp.tile <- (%canonloop) sizes(%ts : i32)
5190
5191 return success();
5192}
5193
5194LogicalResult TileOp::verify() {
5195 if (getApplyees().empty())
5196 return emitOpError() << "must apply to at least one loop";
5197
5198 if (getSizes().size() != getApplyees().size())
5199 return emitOpError() << "there must be one tile size for each applyee";
5200
5201 if (!getGeneratees().empty() &&
5202 2 * getSizes().size() != getGeneratees().size())
5203 return emitOpError()
5204 << "expecting two times the number of generatees than applyees";
5205
5206 return checkApplyeesNesting(op: *this);
5207}
5208
5209std::pair<unsigned, unsigned> TileOp ::getApplyeesODSOperandIndexAndLength() {
5210 return getODSOperandIndexAndLength(index: odsIndex_applyees);
5211}
5212
5213std::pair<unsigned, unsigned> TileOp::getGenerateesODSOperandIndexAndLength() {
5214 return getODSOperandIndexAndLength(index: odsIndex_generatees);
5215}
5216
5217//===----------------------------------------------------------------------===//
5218// FuseOp
5219//===----------------------------------------------------------------------===//
5220
5221static void printLoopTransformClis(OpAsmPrinter &p, FuseOp op,
5222 OperandRange generatees,
5223 OperandRange applyees) {
5224 if (!generatees.empty())
5225 p << '(' << llvm::interleaved(R: generatees) << ')';
5226
5227 if (!applyees.empty())
5228 p << " <- (" << llvm::interleaved(R: applyees) << ')';
5229}
5230
5231LogicalResult FuseOp::verify() {
5232 if (getApplyees().size() < 2)
5233 return emitOpError() << "must apply to at least two loops";
5234
5235 if (getFirst().has_value() && getCount().has_value()) {
5236 int64_t first = getFirst().value();
5237 int64_t count = getCount().value();
5238 if ((unsigned)(first + count - 1) > getApplyees().size())
5239 return emitOpError() << "the numbers of applyees must be at least first "
5240 "minus one plus count attributes";
5241 if (!getGeneratees().empty() &&
5242 getGeneratees().size() != getApplyees().size() + 1 - count)
5243 return emitOpError() << "the number of generatees must be the number of "
5244 "aplyees plus one minus count";
5245
5246 } else {
5247 if (!getGeneratees().empty() && getGeneratees().size() != 1)
5248 return emitOpError()
5249 << "in a complete fuse the number of generatees must be exactly 1";
5250 }
5251 for (auto &&applyee : getApplyees()) {
5252 auto [create, gen, cons] = decodeCli(cli: applyee);
5253
5254 if (!gen)
5255 return emitOpError() << "applyee CLI has no generator";
5256 auto loop = dyn_cast_or_null<CanonicalLoopOp>(Val: gen->getOwner());
5257 if (!loop)
5258 return emitOpError()
5259 << "currently only supports omp.canonical_loop as applyee";
5260 }
5261 return success();
5262}
5263std::pair<unsigned, unsigned> FuseOp::getApplyeesODSOperandIndexAndLength() {
5264 return getODSOperandIndexAndLength(index: odsIndex_applyees);
5265}
5266
5267std::pair<unsigned, unsigned> FuseOp::getGenerateesODSOperandIndexAndLength() {
5268 return getODSOperandIndexAndLength(index: odsIndex_generatees);
5269}
5270
5271//===----------------------------------------------------------------------===//
5272// Critical construct (2.17.1)
5273//===----------------------------------------------------------------------===//
5274
5275void CriticalDeclareOp::build(OpBuilder &builder, OperationState &state,
5276 const CriticalDeclareOperands &clauses) {
5277 CriticalDeclareOp::build(odsBuilder&: builder, odsState&: state, sym_name: clauses.symName,
5278 sym_visibility: clauses.symVisibility, hint: clauses.hint);
5279}
5280
5281LogicalResult CriticalDeclareOp::verify() {
5282 return verifySynchronizationHint(op: *this, hint: getHint());
5283}
5284
5285LogicalResult CriticalOp::verify() {
5286 SymbolRefAttr currentName = getNameAttr();
5287
5288 CriticalOp parentCritical = (*this)->getParentOfType<CriticalOp>();
5289
5290 while (parentCritical) {
5291 SymbolRefAttr parentName = parentCritical.getNameAttr();
5292
5293 if (currentName == parentName) {
5294 if (currentName) {
5295 return emitOpError() << "cannot be nested inside another omp.critical "
5296 "region with the same name ("
5297 << currentName << ")";
5298 } else {
5299 return emitOpError() << "cannot be nested inside another unnamed "
5300 "omp.critical region";
5301 }
5302 }
5303
5304 parentCritical = parentCritical->getParentOfType<CriticalOp>();
5305 }
5306
5307 return success();
5308}
5309
5310LogicalResult CriticalOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
5311 if (getNameAttr()) {
5312 SymbolRefAttr symbolRef = getNameAttr();
5313 auto decl = symbolTable.lookupNearestSymbolFrom<CriticalDeclareOp>(
5314 from: *this, symbol: symbolRef);
5315 if (!decl) {
5316 return emitOpError() << "expected symbol reference " << symbolRef
5317 << " to point to a critical declaration";
5318 }
5319 }
5320
5321 return success();
5322}
5323
5324//===----------------------------------------------------------------------===//
5325// Spec 5.1: Error directive (2.5.4)
5326//===----------------------------------------------------------------------===//
5327
5328LogicalResult ErrorOp::verify() {
5329 if (getMessage() && getMessageExpr())
5330 return emitOpError() << "the message must be provided either as a constant "
5331 "`message` attribute or as a `message_expr` "
5332 "operand, but not both";
5333 return success();
5334}
5335
5336//===----------------------------------------------------------------------===//
5337// Ordered construct
5338//===----------------------------------------------------------------------===//
5339
5340static LogicalResult verifyOrderedParent(Operation &op) {
5341 bool hasRegion = op.getNumRegions() > 0;
5342 auto loopOp = op.getParentOfType<LoopNestOp>();
5343 if (!loopOp) {
5344 if (hasRegion)
5345 return success();
5346
5347 // TODO: Consider if this needs to be the case only for the standalone
5348 // variant of the ordered construct.
5349 return op.emitOpError() << "must be nested inside of a loop";
5350 }
5351
5352 Operation *wrapper = loopOp->getParentOp();
5353 if (auto wsloopOp = dyn_cast<WsloopOp>(Val: wrapper)) {
5354 IntegerAttr orderedAttr = wsloopOp.getOrderedAttr();
5355 if (!orderedAttr)
5356 return op.emitOpError() << "the enclosing worksharing-loop region must "
5357 "have an ordered clause";
5358
5359 if (hasRegion && orderedAttr.getInt() != 0)
5360 return op.emitOpError() << "the enclosing loop's ordered clause must not "
5361 "have a parameter present";
5362
5363 if (!hasRegion && orderedAttr.getInt() == 0)
5364 return op.emitOpError() << "the enclosing loop's ordered clause must "
5365 "have a parameter present";
5366 } else if (!isa<SimdOp>(Val: wrapper)) {
5367 return op.emitOpError() << "must be nested inside of a worksharing, simd "
5368 "or worksharing simd loop";
5369 }
5370 return success();
5371}
5372
5373void OrderedOp::build(OpBuilder &builder, OperationState &state,
5374 const OrderedOperands &clauses) {
5375 OrderedOp::build(odsBuilder&: builder, odsState&: state, doacross_depend_type: clauses.doacrossDependType,
5376 doacross_num_loops: clauses.doacrossNumLoops, doacross_depend_vars: clauses.doacrossDependVars);
5377}
5378
5379LogicalResult OrderedOp::verify() {
5380 if (failed(Result: verifyOrderedParent(op&: **this)))
5381 return failure();
5382
5383 auto wrapper = (*this)->getParentOfType<WsloopOp>();
5384 if (!wrapper || *wrapper.getOrdered() != *getDoacrossNumLoops())
5385 return emitOpError() << "number of variables in depend clause does not "
5386 << "match number of iteration variables in the "
5387 << "doacross loop";
5388
5389 return success();
5390}
5391
5392void OrderedRegionOp::build(OpBuilder &builder, OperationState &state,
5393 const OrderedRegionOperands &clauses) {
5394 OrderedRegionOp::build(odsBuilder&: builder, odsState&: state, par_level_simd: clauses.parLevelSimd);
5395}
5396
5397LogicalResult OrderedRegionOp::verify() { return verifyOrderedParent(op&: **this); }
5398
5399//===----------------------------------------------------------------------===//
5400// TaskwaitOp
5401//===----------------------------------------------------------------------===//
5402
5403void TaskwaitOp::build(OpBuilder &builder, OperationState &state,
5404 const TaskwaitOperands &clauses) {
5405 // TODO Store clauses in op: depend_iterated_kinds, depend_iterated, nowait.
5406 MLIRContext *ctx = builder.getContext();
5407 TaskwaitOp::build(
5408 odsBuilder&: builder, odsState&: state,
5409 /*depend_kinds=*/makeArrayAttr(context: ctx, attrs: clauses.dependKinds),
5410 /*depend_vars=*/clauses.dependVars,
5411 /*depend_iterated_kinds=*/makeArrayAttr(context: ctx, attrs: clauses.dependIteratedKinds),
5412 /*depend_iterated=*/ValueRange(clauses.dependIterated),
5413 /*nowait=*/clauses.nowait);
5414}
5415
5416//===----------------------------------------------------------------------===//
5417// Verifier for AtomicReadOp
5418//===----------------------------------------------------------------------===//
5419
5420LogicalResult AtomicReadOp::verify() {
5421 if (verifyCommon().failed())
5422 return mlir::failure();
5423
5424 int64_t version = 50;
5425 if (auto moduleOp = getOperation()->getParentOfType<ModuleOp>())
5426 if (Attribute verAttr = moduleOp->getDiscardableAttr(name: "omp.version"))
5427 version = llvm::cast<VersionAttr>(Val&: verAttr).getVersion();
5428
5429 if (auto mo = getMemoryOrder()) {
5430 if (*mo == ClauseMemoryOrderKind::Release) {
5431 return emitError(message: "memory-order must not be release for atomic reads");
5432 }
5433 if (*mo == ClauseMemoryOrderKind::Acq_rel) {
5434 // acq_rel is prohibited on read only in OpenMP 5.0; allowed in 5.1+.
5435 if (version < 51)
5436 return emitError(message: "memory-order must not be acq_rel for atomic reads");
5437 }
5438 }
5439 return verifySynchronizationHint(op: *this, hint: getHint());
5440}
5441
5442//===----------------------------------------------------------------------===//
5443// Verifier for AtomicWriteOp
5444//===----------------------------------------------------------------------===//
5445
5446LogicalResult AtomicWriteOp::verify() {
5447 if (verifyCommon().failed())
5448 return mlir::failure();
5449
5450 int64_t version = 50;
5451 if (auto moduleOp = getOperation()->getParentOfType<ModuleOp>())
5452 if (Attribute verAttr = moduleOp->getDiscardableAttr(name: "omp.version"))
5453 version = llvm::cast<VersionAttr>(Val&: verAttr).getVersion();
5454
5455 if (auto mo = getMemoryOrder()) {
5456 if (*mo == ClauseMemoryOrderKind::Acquire) {
5457 return emitError(message: "memory-order must not be acquire for atomic writes");
5458 }
5459 if (*mo == ClauseMemoryOrderKind::Acq_rel) {
5460 // acq_rel is prohibited on write only in OpenMP 5.0; allowed in 5.1+.
5461 if (version < 51)
5462 return emitError(message: "memory-order must not be acq_rel for atomic writes");
5463 }
5464 }
5465 return verifySynchronizationHint(op: *this, hint: getHint());
5466}
5467
5468//===----------------------------------------------------------------------===//
5469// Verifier for AtomicUpdateOp
5470//===----------------------------------------------------------------------===//
5471
5472LogicalResult AtomicUpdateOp::canonicalize(AtomicUpdateOp op,
5473 PatternRewriter &rewriter) {
5474 if (op.isNoOp()) {
5475 rewriter.eraseOp(op);
5476 return success();
5477 }
5478 if (Value writeVal = op.getWriteOpVal()) {
5479 rewriter.replaceOpWithNewOp<AtomicWriteOp>(
5480 op, args: op.getX(), args&: writeVal, args: op.getHintAttr(), args: op.getMemoryOrderAttr());
5481 return success();
5482 }
5483 return failure();
5484}
5485
5486LogicalResult AtomicUpdateOp::verify() {
5487 if (verifyCommon().failed())
5488 return mlir::failure();
5489
5490 int64_t version = 50;
5491 if (auto moduleOp = getOperation()->getParentOfType<ModuleOp>())
5492 if (Attribute verAttr = moduleOp->getDiscardableAttr(name: "omp.version"))
5493 version = llvm::cast<VersionAttr>(Val&: verAttr).getVersion();
5494
5495 if (auto mo = getMemoryOrder()) {
5496 if (*mo == ClauseMemoryOrderKind::Acq_rel ||
5497 *mo == ClauseMemoryOrderKind::Acquire) {
5498 // This restriction applies only to OpenMP 5.0; removed in 5.1.
5499 if (version < 51)
5500 return emitError(
5501 message: "memory-order must not be acq_rel or acquire for atomic updates");
5502 }
5503 }
5504
5505 return verifySynchronizationHint(op: *this, hint: getHint());
5506}
5507
5508LogicalResult AtomicUpdateOp::verifyRegions() { return verifyRegionsCommon(); }
5509
5510//===----------------------------------------------------------------------===//
5511// Verifier for AtomicCaptureOp
5512//===----------------------------------------------------------------------===//
5513
5514AtomicReadOp AtomicCaptureOp::getAtomicReadOp() {
5515 if (auto op = dyn_cast<AtomicReadOp>(Val: getFirstOp()))
5516 return op;
5517 return dyn_cast<AtomicReadOp>(Val: getSecondOp());
5518}
5519
5520AtomicWriteOp AtomicCaptureOp::getAtomicWriteOp() {
5521 if (auto op = dyn_cast<AtomicWriteOp>(Val: getFirstOp()))
5522 return op;
5523 return dyn_cast<AtomicWriteOp>(Val: getSecondOp());
5524}
5525
5526AtomicUpdateOp AtomicCaptureOp::getAtomicUpdateOp() {
5527 if (auto op = dyn_cast<AtomicUpdateOp>(Val: getFirstOp()))
5528 return op;
5529 return dyn_cast<AtomicUpdateOp>(Val: getSecondOp());
5530}
5531
5532AtomicCompareOp AtomicCaptureOp::getAtomicCompareOp() {
5533 if (auto op = dyn_cast<AtomicCompareOp>(Val: getFirstOp()))
5534 return op;
5535 return dyn_cast<AtomicCompareOp>(Val: getSecondOp());
5536}
5537
5538LogicalResult AtomicCaptureOp::verify() {
5539 return verifySynchronizationHint(op: *this, hint: getHint());
5540}
5541
5542LogicalResult AtomicCaptureOp::verifyRegions() {
5543 if (verifyRegionsCommon().failed())
5544 return mlir::failure();
5545
5546 if (getFirstOp()->getInherentAttr(name: "hint").value_or(u: Attribute{}) ||
5547 getSecondOp()->getInherentAttr(name: "hint").value_or(u: Attribute{}))
5548 return emitOpError(
5549 message: "operations inside capture region must not have hint clause");
5550
5551 if (getFirstOp()->getInherentAttr(name: "memory_order").value_or(u: Attribute{}) ||
5552 getSecondOp()->getInherentAttr(name: "memory_order").value_or(u: Attribute{}))
5553 return emitOpError(
5554 message: "operations inside capture region must not have memory_order clause");
5555 return success();
5556}
5557
5558//===----------------------------------------------------------------------===//
5559// AtomicCompareOp
5560//===----------------------------------------------------------------------===//
5561
5562LogicalResult AtomicCompareOp::verify() {
5563 if (verifyCommon().failed())
5564 return mlir::failure();
5565 // OpenMP 5.2 [15.8.3]: the fail clause argument must be one of seq_cst,
5566 // acquire or relaxed ('release' and 'acq_rel' are not valid failure
5567 // orderings and map to invalid cmpxchg failure orderings).
5568 if (auto failOrder = getFailMemoryOrder()) {
5569 if (*failOrder != ClauseMemoryOrderKind::Seq_cst &&
5570 *failOrder != ClauseMemoryOrderKind::Acquire &&
5571 *failOrder != ClauseMemoryOrderKind::Relaxed)
5572 return emitOpError(
5573 message: "fail_memory_order must be 'seq_cst', 'acquire' or 'relaxed'");
5574 }
5575 return verifySynchronizationHint(op: *this, hint: getHint());
5576}
5577
5578LogicalResult AtomicCompareOp::verifyRegions() {
5579 if (verifyRegionsCommon().failed())
5580 return mlir::failure();
5581
5582 if (verifyOperator().failed())
5583 return mlir::failure();
5584
5585 Block &block = getRegion().front();
5586
5587 Operation *terminator = block.getTerminator();
5588 if (!terminator || !isa<YieldOp>(Val: terminator))
5589 return emitOpError(message: "region must be terminated with omp.yield");
5590
5591 return success();
5592}
5593
5594//===----------------------------------------------------------------------===//
5595// CancelOp
5596//===----------------------------------------------------------------------===//
5597
5598void CancelOp::build(OpBuilder &builder, OperationState &state,
5599 const CancelOperands &clauses) {
5600 CancelOp::build(odsBuilder&: builder, odsState&: state, cancel_directive: clauses.cancelDirective, if_expr: clauses.ifExpr);
5601}
5602
5603static Operation *getParentInSameDialect(Operation *thisOp) {
5604 Operation *parent = thisOp->getParentOp();
5605 while (parent) {
5606 if (parent->getDialect() == thisOp->getDialect())
5607 return parent;
5608 parent = parent->getParentOp();
5609 }
5610 return nullptr;
5611}
5612
5613LogicalResult CancelOp::verify() {
5614 ClauseCancellationConstructType cct = getCancelDirective();
5615 // The next OpenMP operation in the chain of parents
5616 Operation *structuralParent = getParentInSameDialect(thisOp: (*this).getOperation());
5617 if (!structuralParent)
5618 return emitOpError() << "Orphaned cancel construct";
5619
5620 if ((cct == ClauseCancellationConstructType::Parallel) &&
5621 !mlir::isa<ParallelOp>(Val: structuralParent)) {
5622 return emitOpError() << "cancel parallel must appear "
5623 << "inside a parallel region";
5624 }
5625 if (cct == ClauseCancellationConstructType::Loop) {
5626 // structural parent will be omp.loop_nest, directly nested inside
5627 // omp.wsloop
5628 auto wsloopOp = mlir::dyn_cast<WsloopOp>(Val: structuralParent->getParentOp());
5629
5630 if (!wsloopOp) {
5631 return emitOpError()
5632 << "cancel loop must appear inside a worksharing-loop region";
5633 }
5634 if (wsloopOp.getNowaitAttr()) {
5635 return emitError() << "A worksharing construct that is canceled "
5636 << "must not have a nowait clause";
5637 }
5638 if (wsloopOp.getOrderedAttr()) {
5639 return emitError() << "A worksharing construct that is canceled "
5640 << "must not have an ordered clause";
5641 }
5642
5643 } else if (cct == ClauseCancellationConstructType::Sections) {
5644 // structural parent will be an omp.section, directly nested inside
5645 // omp.sections
5646 auto sectionsOp =
5647 mlir::dyn_cast<SectionsOp>(Val: structuralParent->getParentOp());
5648 if (!sectionsOp) {
5649 return emitOpError() << "cancel sections must appear "
5650 << "inside a sections region";
5651 }
5652 if (sectionsOp.getNowait()) {
5653 return emitError() << "A sections construct that is canceled "
5654 << "must not have a nowait clause";
5655 }
5656 }
5657 if ((cct == ClauseCancellationConstructType::Taskgroup) &&
5658 (!mlir::isa<omp::TaskOp>(Val: structuralParent) &&
5659 !mlir::isa<omp::TaskloopWrapperOp>(Val: structuralParent->getParentOp()))) {
5660 return emitOpError() << "cancel taskgroup must appear "
5661 << "inside a task region";
5662 }
5663 return success();
5664}
5665
5666//===----------------------------------------------------------------------===//
5667// CancellationPointOp
5668//===----------------------------------------------------------------------===//
5669
5670void CancellationPointOp::build(OpBuilder &builder, OperationState &state,
5671 const CancellationPointOperands &clauses) {
5672 CancellationPointOp::build(odsBuilder&: builder, odsState&: state, cancel_directive: clauses.cancelDirective);
5673}
5674
5675LogicalResult CancellationPointOp::verify() {
5676 ClauseCancellationConstructType cct = getCancelDirective();
5677 // The next OpenMP operation in the chain of parents
5678 Operation *structuralParent = getParentInSameDialect(thisOp: (*this).getOperation());
5679 if (!structuralParent)
5680 return emitOpError() << "Orphaned cancellation point";
5681
5682 if ((cct == ClauseCancellationConstructType::Parallel) &&
5683 !mlir::isa<ParallelOp>(Val: structuralParent)) {
5684 return emitOpError() << "cancellation point parallel must appear "
5685 << "inside a parallel region";
5686 }
5687 // Strucutal parent here will be an omp.loop_nest. Get the parent of that to
5688 // find the wsloop
5689 if ((cct == ClauseCancellationConstructType::Loop) &&
5690 !mlir::isa<WsloopOp>(Val: structuralParent->getParentOp())) {
5691 return emitOpError() << "cancellation point loop must appear "
5692 << "inside a worksharing-loop region";
5693 }
5694 if ((cct == ClauseCancellationConstructType::Sections) &&
5695 !mlir::isa<omp::SectionOp>(Val: structuralParent)) {
5696 return emitOpError() << "cancellation point sections must appear "
5697 << "inside a sections region";
5698 }
5699 if ((cct == ClauseCancellationConstructType::Taskgroup) &&
5700 (!mlir::isa<omp::TaskOp>(Val: structuralParent) &&
5701 !mlir::isa<omp::TaskloopWrapperOp>(Val: structuralParent->getParentOp()))) {
5702 return emitOpError() << "cancellation point taskgroup must appear "
5703 << "inside a task region";
5704 }
5705 return success();
5706}
5707
5708//===----------------------------------------------------------------------===//
5709// MapBoundsOp
5710//===----------------------------------------------------------------------===//
5711
5712LogicalResult MapBoundsOp::verify() {
5713 auto extent = getExtent();
5714 auto upperbound = getUpperBound();
5715 if (!extent && !upperbound)
5716 return emitError(message: "expected extent or upperbound.");
5717 return success();
5718}
5719
5720void PrivateClauseOp::build(OpBuilder &odsBuilder, OperationState &odsState,
5721 TypeRange /*result_types*/, StringAttr symName,
5722 TypeAttr type) {
5723 PrivateClauseOp::build(
5724 odsBuilder, odsState, sym_name: symName, /*sym_visibility=*/nullptr, type,
5725 data_sharing_type: DataSharingClauseTypeAttr::get(context: odsBuilder.getContext(),
5726 value: DataSharingClauseType::Private));
5727}
5728
5729LogicalResult PrivateClauseOp::verifyRegions() {
5730 Type argType = getArgType();
5731 auto verifyTerminator = [&](Operation *terminator, bool yieldsValue,
5732 bool yieldsAnyType) -> LogicalResult {
5733 if (!terminator->getBlock()->getSuccessors().empty())
5734 return success();
5735
5736 if (!llvm::isa<YieldOp>(Val: terminator))
5737 return mlir::emitError(loc: terminator->getLoc())
5738 << "expected exit block terminator to be an `omp.yield` op.";
5739
5740 YieldOp yieldOp = llvm::cast<YieldOp>(Val: terminator);
5741 TypeRange yieldedTypes = yieldOp.getResults().getTypes();
5742
5743 if (!yieldsValue) {
5744 if (yieldedTypes.empty())
5745 return success();
5746
5747 return mlir::emitError(loc: terminator->getLoc())
5748 << "Did not expect any values to be yielded.";
5749 }
5750
5751 if (yieldedTypes.size() == 1 &&
5752 (yieldsAnyType || yieldedTypes.front() == argType))
5753 return success();
5754
5755 auto error = mlir::emitError(loc: yieldOp.getLoc())
5756 << "Invalid yielded value. Expected ";
5757 if (yieldsAnyType)
5758 error << "a single value";
5759 else
5760 error << "type: " << argType;
5761 error << ", got: ";
5762
5763 if (yieldedTypes.empty())
5764 error << "None";
5765 else
5766 error << yieldedTypes;
5767
5768 return error;
5769 };
5770
5771 auto verifyRegion = [&](Region &region, unsigned expectedNumArgs,
5772 StringRef regionName, bool yieldsValue,
5773 bool yieldsAnyType = false) -> LogicalResult {
5774 assert(!region.empty());
5775
5776 if (region.getNumArguments() != expectedNumArgs)
5777 return mlir::emitError(loc: region.getLoc())
5778 << "`" << regionName << "`: " << "expected " << expectedNumArgs
5779 << " region arguments, got: " << region.getNumArguments();
5780
5781 for (Block &block : region) {
5782 // MLIR will verify the absence of the terminator for us.
5783 if (!block.mightHaveTerminator())
5784 continue;
5785
5786 if (failed(Result: verifyTerminator(block.getTerminator(), yieldsValue,
5787 yieldsAnyType)))
5788 return failure();
5789 }
5790
5791 return success();
5792 };
5793
5794 // Ensure all of the region arguments have the same type
5795 for (Region *region : getRegions())
5796 for (Type ty : region->getArgumentTypes())
5797 if (ty != argType)
5798 return emitError() << "Region argument type mismatch: got " << ty
5799 << " expected " << argType << ".";
5800
5801 mlir::Region &initRegion = getInitRegion();
5802 if (!initRegion.empty() &&
5803 failed(Result: verifyRegion(getInitRegion(), /*expectedNumArgs=*/2, "init",
5804 /*yieldsValue=*/true)))
5805 return failure();
5806
5807 DataSharingClauseType dsType = getDataSharingType();
5808
5809 if (dsType == DataSharingClauseType::Private && !getCopyRegion().empty())
5810 return emitError(message: "`private` clauses do not require a `copy` region.");
5811
5812 if (dsType == DataSharingClauseType::FirstPrivate && getCopyRegion().empty())
5813 return emitError(
5814 message: "`firstprivate` clauses require at least a `copy` region.");
5815
5816 if (dsType == DataSharingClauseType::FirstPrivate &&
5817 failed(Result: verifyRegion(getCopyRegion(), /*expectedNumArgs=*/2, "copy",
5818 /*yieldsValue=*/true)))
5819 return failure();
5820
5821 if (dsType == DataSharingClauseType::Linear && !getCopyRegion().empty())
5822 return emitError(message: "`linear` clauses do not require a `copy` region.");
5823
5824 if (!getDataAddressRegion().empty()) {
5825 if (dsType != DataSharingClauseType::Linear)
5826 return emitError(
5827 message: "only `linear` clauses can have a `data_address` region.");
5828 if (failed(Result: verifyRegion(getDataAddressRegion(), /*expectedNumArgs=*/1,
5829 "data_address", /*yieldsValue=*/true,
5830 /*yieldsAnyType=*/true)))
5831 return failure();
5832 }
5833
5834 if (!getDeallocRegion().empty() &&
5835 failed(Result: verifyRegion(getDeallocRegion(), /*expectedNumArgs=*/1, "dealloc",
5836 /*yieldsValue=*/false)))
5837 return failure();
5838
5839 return success();
5840}
5841
5842//===----------------------------------------------------------------------===//
5843// Spec 5.2: Masked construct (10.5)
5844//===----------------------------------------------------------------------===//
5845
5846void MaskedOp::build(OpBuilder &builder, OperationState &state,
5847 const MaskedOperands &clauses) {
5848 MaskedOp::build(odsBuilder&: builder, odsState&: state, filtered_thread_id: clauses.filteredThreadId);
5849}
5850
5851//===----------------------------------------------------------------------===//
5852// Spec 5.2: Dispatch construct (7.6)
5853//===----------------------------------------------------------------------===//
5854
5855void DispatchOp::build(OpBuilder &builder, OperationState &state,
5856 const DispatchOperands &clauses) {
5857 DispatchOp::build(odsBuilder&: builder, odsState&: state, nocontext: clauses.nocontext, novariants: clauses.novariants,
5858 nowait: clauses.nowait);
5859}
5860
5861//===----------------------------------------------------------------------===//
5862// Spec 5.2: Scan construct (5.6)
5863//===----------------------------------------------------------------------===//
5864
5865void ScanOp::build(OpBuilder &builder, OperationState &state,
5866 const ScanOperands &clauses) {
5867 ScanOp::build(odsBuilder&: builder, odsState&: state, inclusive_vars: clauses.inclusiveVars, exclusive_vars: clauses.exclusiveVars);
5868}
5869
5870LogicalResult ScanOp::verify() {
5871 if (hasExclusiveVars() == hasInclusiveVars())
5872 return emitError(
5873 message: "Exactly one of EXCLUSIVE or INCLUSIVE clause is expected");
5874 if (WsloopOp parentWsLoopOp = (*this)->getParentOfType<WsloopOp>()) {
5875 if (parentWsLoopOp.hasReductionModifier(modifier: ReductionModifier::inscan))
5876 return success();
5877 }
5878 if (SimdOp parentSimdOp = (*this)->getParentOfType<SimdOp>()) {
5879 if (parentSimdOp.hasReductionModifier(modifier: ReductionModifier::inscan))
5880 return success();
5881 }
5882 return emitError(message: "SCAN directive needs to be enclosed within a parent "
5883 "worksharing loop construct or SIMD construct with INSCAN "
5884 "reduction modifier");
5885}
5886
5887/// Verifies align clause in allocate directive
5888LogicalResult verifyAlignment(Operation &op,
5889 std::optional<uint64_t> alignment) {
5890 if (alignment.has_value()) {
5891 if ((alignment.value() != 0) && !llvm::has_single_bit(Value: alignment.value()))
5892 return op.emitError()
5893 << "ALIGN value : " << alignment.value() << " must be power of 2";
5894 }
5895 return success();
5896}
5897
5898LogicalResult AllocateDirOp::verify() {
5899 return verifyAlignment(op&: *getOperation(), alignment: getAlign());
5900}
5901
5902//===----------------------------------------------------------------------===//
5903// AllocSharedMemOp
5904//===----------------------------------------------------------------------===//
5905
5906LogicalResult AllocSharedMemOp::verify() {
5907 return verifyAlignment(op&: *getOperation(), alignment: getMemAlignment());
5908}
5909
5910//===----------------------------------------------------------------------===//
5911// FreeSharedMemOp
5912//===----------------------------------------------------------------------===//
5913
5914LogicalResult FreeSharedMemOp::verify() {
5915 return verifyAlignment(op&: *getOperation(), alignment: getMemAlignment());
5916}
5917
5918//===----------------------------------------------------------------------===//
5919// WorkdistributeOp
5920//===----------------------------------------------------------------------===//
5921
5922LogicalResult WorkdistributeOp::verify() {
5923 if (isCombined())
5924 return emitOpError() << "cannot be a non-innermost combined construct leaf";
5925
5926 // Check that region exists and is not empty
5927 Region &region = getRegion();
5928 if (region.empty())
5929 return emitOpError(message: "region cannot be empty");
5930 // Verify single entry point.
5931 Block &entryBlock = region.front();
5932 if (entryBlock.empty())
5933 return emitOpError(message: "region must contain a structured block");
5934 // Verify single exit point.
5935 bool hasTerminator = false;
5936 for (Block &block : region) {
5937 if (isa<TerminatorOp>(Val: block.back())) {
5938 if (hasTerminator) {
5939 return emitOpError(message: "region must have exactly one terminator");
5940 }
5941 hasTerminator = true;
5942 }
5943 }
5944 if (!hasTerminator) {
5945 return emitOpError(message: "region must be terminated with omp.terminator");
5946 }
5947 auto walkResult = region.walk(callback: [&](Operation *op) -> WalkResult {
5948 // No implicit barrier at end
5949 if (isa<BarrierOp>(Val: op)) {
5950 return emitOpError(
5951 message: "explicit barriers are not allowed in workdistribute region");
5952 }
5953 // Check for invalid nested constructs
5954 if (isa<ParallelOp>(Val: op)) {
5955 return emitOpError(
5956 message: "nested parallel constructs not allowed in workdistribute");
5957 }
5958 if (isa<TeamsOp>(Val: op)) {
5959 return emitOpError(
5960 message: "nested teams constructs not allowed in workdistribute");
5961 }
5962 return WalkResult::advance();
5963 });
5964 if (walkResult.wasInterrupted())
5965 return failure();
5966
5967 Operation *parentOp = (*this)->getParentOp();
5968 if (!llvm::dyn_cast<TeamsOp>(Val: parentOp))
5969 return emitOpError(message: "workdistribute must be nested under teams");
5970 return success();
5971}
5972
5973//===----------------------------------------------------------------------===//
5974// Declare simd [7.7]
5975//===----------------------------------------------------------------------===//
5976
5977LogicalResult DeclareSimdOp::verify() {
5978 // Must be nested inside a function-like op
5979 auto func =
5980 dyn_cast_if_present<mlir::FunctionOpInterface>(Val: (*this)->getParentOp());
5981 if (!func)
5982 return emitOpError() << "must be nested inside a function";
5983
5984 if (getInbranch() && getNotinbranch())
5985 return emitOpError(message: "cannot have both 'inbranch' and 'notinbranch'");
5986
5987 if (failed(Result: verifyLinearModifiers(op: *this, linearModifiers: getLinearModifiers(), linearVars: getLinearVars(),
5988 /*isDeclareSimd=*/true)))
5989 return failure();
5990
5991 if (getLinearSyms())
5992 return emitOpError(message: "linear privatizers are not allowed");
5993
5994 return verifyAlignedClause(op: *this, alignments: getAlignments(), alignedVars: getAlignedVars());
5995}
5996
5997void DeclareSimdOp::build(OpBuilder &odsBuilder, OperationState &odsState,
5998 const DeclareSimdOperands &clauses) {
5999 MLIRContext *ctx = odsBuilder.getContext();
6000 DeclareSimdOp::build(
6001 odsBuilder, odsState, aligned_vars: clauses.alignedVars,
6002 alignments: makeArrayAttr(context: ctx, attrs: clauses.alignments), inbranch: clauses.inbranch,
6003 linear_vars: clauses.linearVars, linear_step_vars: clauses.linearStepVars, linear_var_types: clauses.linearVarTypes,
6004 linear_modifiers: clauses.linearModifiers, linear_syms: makeArrayAttr(context: ctx, attrs: clauses.linearSyms),
6005 notinbranch: clauses.notinbranch, simdlen: clauses.simdlen, uniform_vars: clauses.uniformVars);
6006}
6007
6008//===----------------------------------------------------------------------===//
6009// Parser and printer for Uniform Clause
6010//===----------------------------------------------------------------------===//
6011
6012/// uniform ::= `uniform` `(` uniform-list `)`
6013/// uniform-list := uniform-val (`,` uniform-val)*
6014/// uniform-val := ssa-id `:` type
6015static ParseResult
6016parseUniformClause(OpAsmParser &parser,
6017 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &uniformVars,
6018 SmallVectorImpl<Type> &uniformTypes) {
6019 return parser.parseCommaSeparatedList(parseElementFn: [&]() -> mlir::ParseResult {
6020 if (parser.parseOperand(result&: uniformVars.emplace_back()) ||
6021 parser.parseColonType(result&: uniformTypes.emplace_back()))
6022 return mlir::failure();
6023 return mlir::success();
6024 });
6025}
6026
6027/// Print Uniform Clauses
6028static void printUniformClause(OpAsmPrinter &p, Operation *op,
6029 ValueRange uniformVars, TypeRange uniformTypes) {
6030 for (unsigned i = 0; i < uniformVars.size(); ++i) {
6031 if (i != 0)
6032 p << ", ";
6033 p << uniformVars[i] << " : " << uniformTypes[i];
6034 }
6035}
6036
6037//===----------------------------------------------------------------------===//
6038// Parser and printer for Affinity Clause
6039//===----------------------------------------------------------------------===//
6040
6041static ParseResult parseAffinityClause(
6042 OpAsmParser &parser,
6043 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &iterated,
6044 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &affinityVars,
6045 SmallVectorImpl<Type> &iteratedTypes,
6046 SmallVectorImpl<Type> &affinityVarTypes) {
6047 if (failed(Result: parseSplitIteratedList(
6048 parser, iteratedVars&: iterated, iteratedTypes, plainVars&: affinityVars, plainTypes&: affinityVarTypes,
6049 /*parsePrefix=*/[&]() -> ParseResult { return success(); })))
6050 return failure();
6051 return success();
6052}
6053
6054static void printAffinityClause(OpAsmPrinter &p, Operation *op,
6055 ValueRange iterated, ValueRange affinityVars,
6056 TypeRange iteratedTypes,
6057 TypeRange affinityVarTypes) {
6058 auto nop = [&](Value, Type) {};
6059 printSplitIteratedList(p, iteratedVars: iterated, iteratedTypes, plainVars: affinityVars,
6060 plainTypes: affinityVarTypes,
6061 /*plain prefix*/ printPrefixForPlain&: nop,
6062 /*iterated prefix*/ printPrefixForIterated&: nop);
6063}
6064
6065//===----------------------------------------------------------------------===//
6066// Parser, printer, and verifier for Iterator modifier
6067//===----------------------------------------------------------------------===//
6068
6069static ParseResult
6070parseIteratorHeader(OpAsmParser &parser, Region &region,
6071 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &lbs,
6072 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &ubs,
6073 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &steps,
6074 SmallVectorImpl<Type> &lbTypes,
6075 SmallVectorImpl<Type> &ubTypes,
6076 SmallVectorImpl<Type> &stepTypes) {
6077
6078 llvm::SMLoc ivLoc = parser.getCurrentLocation();
6079 SmallVector<OpAsmParser::Argument> ivArgs;
6080
6081 // Parse induction variables: %i : i32, %j : i32
6082 if (parser.parseCommaSeparatedList(parseElementFn: [&]() -> ParseResult {
6083 OpAsmParser::Argument &arg = ivArgs.emplace_back();
6084 if (parser.parseArgument(result&: arg))
6085 return failure();
6086
6087 // Optional type, default to Index if not provided
6088 if (succeeded(Result: parser.parseOptionalColon())) {
6089 if (parser.parseType(result&: arg.type))
6090 return failure();
6091 } else {
6092 arg.type = parser.getBuilder().getIndexType();
6093 }
6094 return success();
6095 }))
6096 return failure();
6097
6098 // ) = (
6099 if (parser.parseRParen() || parser.parseEqual() || parser.parseLParen())
6100 return failure();
6101
6102 // Parse Ranges: (%lb to %ub step %st, ...)
6103 if (parser.parseCommaSeparatedList(parseElementFn: [&]() -> ParseResult {
6104 OpAsmParser::UnresolvedOperand lb, ub, st;
6105 if (parser.parseOperand(result&: lb) || parser.parseKeyword(keyword: "to") ||
6106 parser.parseOperand(result&: ub) || parser.parseKeyword(keyword: "step") ||
6107 parser.parseOperand(result&: st))
6108 return failure();
6109
6110 lbs.push_back(Elt: lb);
6111 ubs.push_back(Elt: ub);
6112 steps.push_back(Elt: st);
6113 return success();
6114 }))
6115 return failure();
6116
6117 if (parser.parseRParen())
6118 return failure();
6119
6120 if (ivArgs.size() != lbs.size())
6121 return parser.emitError(loc: ivLoc)
6122 << "mismatch: " << ivArgs.size() << " variables but " << lbs.size()
6123 << " ranges";
6124
6125 for (auto &arg : ivArgs) {
6126 lbTypes.push_back(Elt: arg.type);
6127 ubTypes.push_back(Elt: arg.type);
6128 stepTypes.push_back(Elt: arg.type);
6129 }
6130
6131 return parser.parseRegion(region, arguments: ivArgs);
6132}
6133
6134static void printIteratorHeader(OpAsmPrinter &p, Operation *op, Region &region,
6135 ValueRange lbs, ValueRange ubs,
6136 ValueRange steps, TypeRange, TypeRange,
6137 TypeRange) {
6138 Block &entry = region.front();
6139
6140 for (unsigned i = 0, e = entry.getNumArguments(); i < e; ++i) {
6141 if (i != 0)
6142 p << ", ";
6143 p.printRegionArgument(arg: entry.getArgument(i));
6144 }
6145 p << ") = (";
6146
6147 // (%lb0 to %ub0 step %step0, %lb1 to %ub1 step %step1, ...)
6148 for (unsigned i = 0, e = lbs.size(); i < e; ++i) {
6149 if (i)
6150 p << ", ";
6151 p << lbs[i] << " to " << ubs[i] << " step " << steps[i];
6152 }
6153 p << ") ";
6154
6155 p.printRegion(blocks&: region, /*printEntryBlockArgs=*/false,
6156 /*printBlockTerminators=*/true);
6157}
6158
6159YieldOp IteratorOp::getYieldOp() {
6160 YieldOp yield;
6161 for (Block &block : getRegion()) {
6162 auto blockYield =
6163 dyn_cast_if_present<YieldOp>(Val: block.empty() ? nullptr : &block.back());
6164 if (!blockYield)
6165 continue;
6166 if (yield)
6167 return {};
6168 yield = blockYield;
6169 }
6170 return yield;
6171}
6172
6173LogicalResult IteratorOp::verify() {
6174 auto iteratedTy = llvm::dyn_cast<omp::IteratedType>(Val: getIterated().getType());
6175 if (!iteratedTy)
6176 return emitOpError() << "result must be omp.iterated<entry_ty>";
6177
6178 Block::BlockArgListType ivs = getRegion().front().getArguments();
6179 if (getLoopLowerBounds().size() != ivs.size())
6180 return emitOpError() << "number of range arguments and IVs do not match";
6181
6182 for (auto [lb, iv] : llvm::zip_equal(t: getLoopLowerBounds(), u&: ivs)) {
6183 if (lb.getType() != iv.getType())
6184 return emitOpError()
6185 << "range argument type does not match corresponding IV type";
6186 }
6187
6188 for (Value step : getLoopSteps()) {
6189 if (matchPattern(value: step, pattern: m_Zero()))
6190 return emitOpError() << "loop step must not be zero";
6191 }
6192
6193 YieldOp yield = getYieldOp();
6194 if (!yield) {
6195 bool hasYield = llvm::any_of(Range&: getRegion(), P: [](Block &block) {
6196 return !block.empty() && isa<YieldOp>(Val: block.back());
6197 });
6198 if (hasYield)
6199 return emitOpError() << "region must contain exactly one omp.yield";
6200 return emitOpError() << "region must be terminated by omp.yield";
6201 }
6202
6203 if (yield.getNumOperands() != 1)
6204 return emitOpError()
6205 << "omp.yield in omp.iterator region must yield exactly one value";
6206
6207 mlir::Type yieldedTy = yield.getOperand(i: 0).getType();
6208 mlir::Type elemTy = iteratedTy.getElementType();
6209
6210 if (yieldedTy != elemTy)
6211 return emitOpError() << "omp.iterated element type (" << elemTy
6212 << ") does not match omp.yield operand type ("
6213 << yieldedTy << ")";
6214
6215 return success();
6216}
6217
6218LogicalResult IteratorOp::verifyRegions() {
6219 // Translation routes only the omp.yield block back to the iterator loop.
6220 for (Block &block : getRegion()) {
6221 Operation *terminator = block.getTerminator();
6222 if (block.getNumSuccessors() == 0 && !isa<YieldOp>(Val: terminator))
6223 return mlir::emitError(loc: terminator->getLoc())
6224 << "expected exit block terminator to be an `omp.yield` op.";
6225 }
6226 return success();
6227}
6228
6229//===----------------------------------------------------------------------===//
6230// GroupprivateOp
6231//===----------------------------------------------------------------------===//
6232
6233LogicalResult
6234GroupprivateOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
6235 auto *symbol = symbolTable.lookupNearestSymbolFrom(from: *this, symbol: getSymNameAttr());
6236 if (!symbol)
6237 return emitOpError() << "expected symbol reference '" << getSymName()
6238 << "' to point to a global variable";
6239
6240 if (isa<FunctionOpInterface>(Val: symbol))
6241 return emitOpError() << "expected symbol reference '" << getSymName()
6242 << "' to point to a global variable, not a function";
6243
6244 return success();
6245}
6246
6247#define GET_ATTRDEF_CLASSES
6248#include "mlir/Dialect/OpenMP/OpenMPOpsAttributes.cpp.inc"
6249
6250#define GET_OP_CLASSES
6251#include "mlir/Dialect/OpenMP/OpenMPOps.cpp.inc"
6252
6253#define GET_TYPEDEF_CLASSES
6254#include "mlir/Dialect/OpenMP/OpenMPOpsTypes.cpp.inc"
6255