| 1 | //===- OpenACCCG.cpp - OpenACC codegen ops, attributes, and types ---------===// |
| 2 | // |
| 3 | // Part of the MLIR Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // Implementation for OpenACC codegen operations, attributes, and types. |
| 10 | // These correspond to the definitions in OpenACCCG*.td tablegen files |
| 11 | // and are kept in a separate file because they do not represent direct mappings |
| 12 | // of OpenACC language constructs; they are intermediate representations used |
| 13 | // when decomposing and lowering primary `acc` dialect operations. |
| 14 | // |
| 15 | //===----------------------------------------------------------------------===// |
| 16 | |
| 17 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 18 | #include "mlir/Dialect/GPU/IR/GPUBase.h" |
| 19 | #include "mlir/Dialect/OpenACC/OpenACC.h" |
| 20 | #include "mlir/Dialect/Utils/StaticValueUtils.h" |
| 21 | #include "mlir/IR/BuiltinAttributes.h" |
| 22 | #include "mlir/IR/BuiltinTypes.h" |
| 23 | #include "mlir/IR/PatternMatch.h" |
| 24 | #include "mlir/IR/Region.h" |
| 25 | #include "mlir/Interfaces/ControlFlowInterfaces.h" |
| 26 | #include "mlir/Support/LogicalResult.h" |
| 27 | #include "mlir/Transforms/RegionUtils.h" |
| 28 | #include "llvm/ADT/DenseMap.h" |
| 29 | #include "llvm/ADT/STLExtras.h" |
| 30 | #include "llvm/ADT/SmallVector.h" |
| 31 | |
| 32 | using namespace mlir; |
| 33 | using namespace acc; |
| 34 | |
| 35 | namespace { |
| 36 | /// Generic helper for single-region OpenACC ops that execute their body once |
| 37 | /// and then continue after the operation with their results (if any). |
| 38 | static void |
| 39 | getSingleRegionOpSuccessorRegions(Operation *op, Region ®ion, |
| 40 | RegionBranchPoint point, |
| 41 | SmallVectorImpl<RegionSuccessor> ®ions) { |
| 42 | if (point.isParent()) { |
| 43 | regions.push_back(Elt: RegionSuccessor(®ion)); |
| 44 | return; |
| 45 | } |
| 46 | regions.push_back(Elt: RegionSuccessor(op)); |
| 47 | } |
| 48 | |
| 49 | static ValueRange getSingleRegionSuccessorInputs(Operation *op, |
| 50 | RegionSuccessor successor) { |
| 51 | return successor.isOperation() ? ValueRange(op->getResults()) : ValueRange(); |
| 52 | } |
| 53 | |
| 54 | /// Remove empty acc.kernel_environment operations. If the operation has wait |
| 55 | /// operands, create a acc.wait operation to preserve synchronization. |
| 56 | struct RemoveEmptyKernelEnvironment |
| 57 | : public OpRewritePattern<acc::KernelEnvironmentOp> { |
| 58 | using OpRewritePattern<acc::KernelEnvironmentOp>::OpRewritePattern; |
| 59 | |
| 60 | LogicalResult matchAndRewrite(acc::KernelEnvironmentOp op, |
| 61 | PatternRewriter &rewriter) const override { |
| 62 | assert(op->getNumRegions() == 1 && "expected op to have one region" ); |
| 63 | |
| 64 | Block &block = op.getRegion().front(); |
| 65 | if (!block.empty()) |
| 66 | return failure(); |
| 67 | |
| 68 | // Remove empty kernel environment. |
| 69 | // Preserve synchronization by creating acc.wait operation if needed. |
| 70 | if (!op.getWaitOperands().empty() || op.getWaitOnlyAttr()) |
| 71 | rewriter.replaceOpWithNewOp<acc::WaitOp>( |
| 72 | op, args: op.getWaitOperands(), /*asyncOperand=*/args: Value(), |
| 73 | args: op.getWaitDevnum(), /*async=*/args: nullptr, /*ifCond=*/args: Value()); |
| 74 | else |
| 75 | rewriter.eraseOp(op); |
| 76 | |
| 77 | return success(); |
| 78 | } |
| 79 | }; |
| 80 | |
| 81 | struct KernelEnvironmentDedupDataOperands |
| 82 | : public OpRewritePattern<acc::KernelEnvironmentOp> { |
| 83 | using OpRewritePattern<acc::KernelEnvironmentOp>::OpRewritePattern; |
| 84 | |
| 85 | static bool isReductionDataOperand(Value operand) { |
| 86 | Operation *def = operand.getDefiningOp(); |
| 87 | if (!def) |
| 88 | return false; |
| 89 | std::optional<DataClause> clause = getDataClause(accDataEntryOp: def); |
| 90 | if (!clause || *clause != DataClause::acc_reduction) |
| 91 | return false; |
| 92 | assert(isa<CopyinOp>(def) && "reduction clause must be an acc.copyin" ); |
| 93 | return true; |
| 94 | } |
| 95 | |
| 96 | LogicalResult matchAndRewrite(acc::KernelEnvironmentOp op, |
| 97 | PatternRewriter &rewriter) const override { |
| 98 | SmallVector<Value> operands(op.getDataClauseOperands()); |
| 99 | const unsigned numOperands = operands.size(); |
| 100 | if (numOperands < 2) |
| 101 | return failure(); |
| 102 | |
| 103 | SmallVector<Value> kept; |
| 104 | SmallVector<std::pair<Value, Value>> useRewrites; |
| 105 | SmallVector<SmallVector<Value>> boundsStorage; |
| 106 | boundsStorage.reserve(N: numOperands); |
| 107 | llvm::DenseMap<std::pair<Value, ArrayRef<Value>>, unsigned> |
| 108 | keptIndexByVarAndBounds; |
| 109 | for (Value operand : operands) { |
| 110 | Operation *def = operand.getDefiningOp(); |
| 111 | // acc.map_info is a finalized mapping. Its order and semantics stand; |
| 112 | // matching only its variable and bounds would ignore flags and |
| 113 | // descriptors. |
| 114 | if (!def || isa<MapInfoOp>(Val: def)) { |
| 115 | kept.push_back(Elt: operand); |
| 116 | continue; |
| 117 | } |
| 118 | |
| 119 | Value var = getVar(accDataClauseOp: def); |
| 120 | if (!var) { |
| 121 | kept.push_back(Elt: operand); |
| 122 | continue; |
| 123 | } |
| 124 | |
| 125 | boundsStorage.push_back(Elt: getBounds(accDataClauseOp: def)); |
| 126 | auto [it, inserted] = keptIndexByVarAndBounds.try_emplace( |
| 127 | Key: std::make_pair(x&: var, y: ArrayRef<Value>(boundsStorage.back())), |
| 128 | Args: kept.size()); |
| 129 | if (inserted) { |
| 130 | kept.push_back(Elt: operand); |
| 131 | continue; |
| 132 | } |
| 133 | |
| 134 | Value &survivor = kept[it->second]; |
| 135 | // A reduction acc.copyin is that copyin plus the reduction. Keeping |
| 136 | // another clause would drop the reduction. |
| 137 | if (isReductionDataOperand(operand) && !isReductionDataOperand(operand: survivor)) |
| 138 | std::swap(a&: survivor, b&: operand); |
| 139 | useRewrites.emplace_back(Args&: operand, Args&: survivor); |
| 140 | } |
| 141 | if (kept.size() == numOperands) |
| 142 | return failure(); |
| 143 | |
| 144 | for (auto [from, to] : useRewrites) { |
| 145 | rewriter.replaceUsesWithIf(from, to, functor: [&](OpOperand &use) { |
| 146 | return op->isProperAncestor(other: use.getOwner()); |
| 147 | }); |
| 148 | } |
| 149 | rewriter.modifyOpInPlace( |
| 150 | root: op, callable: [&] { op.getDataClauseOperandsMutable().assign(values: kept); }); |
| 151 | return success(); |
| 152 | } |
| 153 | }; |
| 154 | |
| 155 | // Capture `hasStream` before erasing inputs because segment metadata is only |
| 156 | // repaired at the end of the modification transaction. |
| 157 | static void setComputeRegionInputOperandSegments(ComputeRegionOp op, |
| 158 | PatternRewriter &rewriter, |
| 159 | size_t numInput, |
| 160 | bool hasStream) { |
| 161 | const size_t numLaunch = op.getLaunchArgs().size(); |
| 162 | op->setInherentAttr( |
| 163 | name: rewriter.getStringAttr(bytes: ComputeRegionOp::getOperandSegmentSizeAttr()), |
| 164 | value: rewriter.getDenseI32ArrayAttr(values: {static_cast<int32_t>(numLaunch), |
| 165 | static_cast<int32_t>(numInput), |
| 166 | hasStream ? 1 : 0})); |
| 167 | } |
| 168 | |
| 169 | struct ComputeRegionRemoveDuplicateArgs |
| 170 | : public OpRewritePattern<ComputeRegionOp> { |
| 171 | using Base::Base; |
| 172 | |
| 173 | LogicalResult matchAndRewrite(ComputeRegionOp op, |
| 174 | PatternRewriter &rewriter) const override { |
| 175 | Block *body = op.getBody(); |
| 176 | const size_t numLaunch = op.getLaunchArgs().size(); |
| 177 | size_t numInput = op.getInputArgs().size(); |
| 178 | assert(body->getNumArguments() == numLaunch + numInput && |
| 179 | "region args mismatch" ); |
| 180 | |
| 181 | bool hasDuplicate = false; |
| 182 | for (size_t j = 1; j < numInput && !hasDuplicate; ++j) |
| 183 | for (size_t i = 0; i < j; ++i) |
| 184 | if (op->getOperand(idx: static_cast<unsigned>(numLaunch + i)) == |
| 185 | op->getOperand(idx: static_cast<unsigned>(numLaunch + j))) { |
| 186 | hasDuplicate = true; |
| 187 | break; |
| 188 | } |
| 189 | if (!hasDuplicate) |
| 190 | return failure(); |
| 191 | |
| 192 | const bool hasStream = static_cast<bool>(op.getStream()); |
| 193 | rewriter.modifyOpInPlace(root: op, callable: [&] { |
| 194 | while (true) { |
| 195 | bool merged = false; |
| 196 | for (size_t j = 1; j < numInput && !merged; ++j) { |
| 197 | for (size_t i = 0; i < j; ++i) { |
| 198 | if (op->getOperand(idx: static_cast<unsigned>(numLaunch + i)) != |
| 199 | op->getOperand(idx: static_cast<unsigned>(numLaunch + j))) |
| 200 | continue; |
| 201 | unsigned keepIdx = static_cast<unsigned>(numLaunch + i); |
| 202 | unsigned dropIdx = static_cast<unsigned>(numLaunch + j); |
| 203 | rewriter.replaceAllUsesWith(from: body->getArgument(i: dropIdx), |
| 204 | to: body->getArgument(i: keepIdx)); |
| 205 | body->eraseArgument(index: dropIdx); |
| 206 | op->eraseOperand(idx: dropIdx); |
| 207 | --numInput; |
| 208 | merged = true; |
| 209 | break; |
| 210 | } |
| 211 | } |
| 212 | if (!merged) |
| 213 | break; |
| 214 | } |
| 215 | setComputeRegionInputOperandSegments(op, rewriter, numInput, hasStream); |
| 216 | }); |
| 217 | return success(); |
| 218 | } |
| 219 | }; |
| 220 | |
| 221 | struct ComputeRegionRemoveUnusedArgs |
| 222 | : public OpRewritePattern<ComputeRegionOp> { |
| 223 | using Base::Base; |
| 224 | |
| 225 | LogicalResult matchAndRewrite(ComputeRegionOp op, |
| 226 | PatternRewriter &rewriter) const override { |
| 227 | Block *body = op.getBody(); |
| 228 | const size_t numLaunch = op.getLaunchArgs().size(); |
| 229 | size_t numInput = op.getInputArgs().size(); |
| 230 | assert(body->getNumArguments() == numLaunch + numInput && |
| 231 | "region args mismatch" ); |
| 232 | |
| 233 | bool hasUnused = false; |
| 234 | for (size_t k = numLaunch; k < numLaunch + numInput; ++k) |
| 235 | if (body->getArgument(i: static_cast<unsigned>(k)).use_empty()) { |
| 236 | hasUnused = true; |
| 237 | break; |
| 238 | } |
| 239 | if (!hasUnused) |
| 240 | return failure(); |
| 241 | |
| 242 | const bool hasStream = static_cast<bool>(op.getStream()); |
| 243 | rewriter.modifyOpInPlace(root: op, callable: [&] { |
| 244 | for (size_t k = numLaunch; k < numLaunch + numInput;) { |
| 245 | if (!body->getArgument(i: static_cast<unsigned>(k)).use_empty()) { |
| 246 | ++k; |
| 247 | continue; |
| 248 | } |
| 249 | body->eraseArgument(index: static_cast<unsigned>(k)); |
| 250 | op->eraseOperand(idx: static_cast<unsigned>(k)); |
| 251 | --numInput; |
| 252 | } |
| 253 | setComputeRegionInputOperandSegments(op, rewriter, numInput, hasStream); |
| 254 | }); |
| 255 | return success(); |
| 256 | } |
| 257 | }; |
| 258 | |
| 259 | template <typename EffectTy> |
| 260 | static void addOperandEffect( |
| 261 | SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>> |
| 262 | &effects, |
| 263 | const MutableOperandRange &operand) { |
| 264 | for (unsigned i = 0, e = operand.size(); i < e; ++i) |
| 265 | effects.emplace_back(EffectTy::get(), &operand[i]); |
| 266 | } |
| 267 | |
| 268 | template <typename EffectTy> |
| 269 | static void addResultEffect( |
| 270 | SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>> |
| 271 | &effects, |
| 272 | Value result) { |
| 273 | effects.emplace_back(EffectTy::get(), mlir::cast<mlir::OpResult>(Val&: result)); |
| 274 | } |
| 275 | |
| 276 | static int64_t gpuProcessorIndex(gpu::Processor p) { |
| 277 | switch (p) { |
| 278 | case gpu::Processor::Sequential: |
| 279 | return 0; |
| 280 | case gpu::Processor::ThreadX: |
| 281 | return 1; |
| 282 | case gpu::Processor::ThreadY: |
| 283 | return 2; |
| 284 | case gpu::Processor::ThreadZ: |
| 285 | return 3; |
| 286 | case gpu::Processor::BlockX: |
| 287 | return 4; |
| 288 | case gpu::Processor::BlockY: |
| 289 | return 5; |
| 290 | case gpu::Processor::BlockZ: |
| 291 | return 6; |
| 292 | } |
| 293 | llvm_unreachable("unhandled gpu::Processor" ); |
| 294 | } |
| 295 | |
| 296 | static gpu::Processor indexToGpuProcessor(int64_t idx) { |
| 297 | switch (idx) { |
| 298 | case 0: |
| 299 | return gpu::Processor::Sequential; |
| 300 | case 1: |
| 301 | return gpu::Processor::ThreadX; |
| 302 | case 2: |
| 303 | return gpu::Processor::ThreadY; |
| 304 | case 3: |
| 305 | return gpu::Processor::ThreadZ; |
| 306 | case 4: |
| 307 | return gpu::Processor::BlockX; |
| 308 | case 5: |
| 309 | return gpu::Processor::BlockY; |
| 310 | case 6: |
| 311 | return gpu::Processor::BlockZ; |
| 312 | default: |
| 313 | return gpu::Processor::Sequential; |
| 314 | } |
| 315 | } |
| 316 | |
| 317 | static GPUParallelDimAttr intToParDim(MLIRContext *context, int64_t dimInt) { |
| 318 | return GPUParallelDimAttr::get( |
| 319 | context, value: IntegerAttr::get(type: IndexType::get(context), value: dimInt)); |
| 320 | } |
| 321 | |
| 322 | static GPUParallelDimAttr processorParDim(MLIRContext *context, |
| 323 | gpu::Processor proc) { |
| 324 | return GPUParallelDimAttr::get( |
| 325 | context, |
| 326 | value: IntegerAttr::get(type: IndexType::get(context), value: gpuProcessorIndex(p: proc))); |
| 327 | } |
| 328 | |
| 329 | static ParseResult parseProcessorValue(AsmParser &parser, |
| 330 | GPUParallelDimAttr &dim) { |
| 331 | std::string keyword; |
| 332 | llvm::SMLoc loc = parser.getCurrentLocation(); |
| 333 | if (failed(Result: parser.parseKeywordOrString(result: &keyword))) |
| 334 | return failure(); |
| 335 | auto maybeProcessor = gpu::symbolizeProcessor(keyword); |
| 336 | if (!maybeProcessor) |
| 337 | return parser.emitError(loc) |
| 338 | << "expected one of ::mlir::gpu::Processor enum names" ; |
| 339 | dim = intToParDim(context: parser.getContext(), dimInt: gpuProcessorIndex(p: *maybeProcessor)); |
| 340 | return success(); |
| 341 | } |
| 342 | |
| 343 | static void printProcessorValue(AsmPrinter &printer, |
| 344 | const GPUParallelDimAttr &attr) { |
| 345 | gpu::Processor processor = indexToGpuProcessor(idx: attr.getValue().getInt()); |
| 346 | printer << gpu::stringifyProcessor(processor); |
| 347 | } |
| 348 | |
| 349 | static FailureOr<SmallVector<GPUParallelDimAttr>> |
| 350 | parseGPUParallelDimList(AsmParser &parser) { |
| 351 | SmallVector<GPUParallelDimAttr> parDims; |
| 352 | auto parseParDim = [&]() -> ParseResult { |
| 353 | GPUParallelDimAttr dim; |
| 354 | if (parseProcessorValue(parser, dim)) |
| 355 | return failure(); |
| 356 | parDims.push_back(Elt: dim); |
| 357 | return success(); |
| 358 | }; |
| 359 | if (parser.parseCommaSeparatedList(delimiter: AsmParser::Delimiter::Square, parseElementFn: parseParDim, |
| 360 | contextMessage: "list of OpenACC GPU parallel dimensions" )) |
| 361 | return failure(); |
| 362 | return parDims; |
| 363 | } |
| 364 | |
| 365 | static void printGPUParallelDimList(AsmPrinter &printer, |
| 366 | ArrayRef<GPUParallelDimAttr> dims) { |
| 367 | printer << "[" ; |
| 368 | llvm::interleaveComma(c: dims, os&: printer, each_fn: [&printer](const GPUParallelDimAttr &p) { |
| 369 | printProcessorValue(printer, attr: p); |
| 370 | }); |
| 371 | printer << "]" ; |
| 372 | } |
| 373 | } // namespace |
| 374 | |
| 375 | //===----------------------------------------------------------------------===// |
| 376 | // KernelEnvironmentOp |
| 377 | //===----------------------------------------------------------------------===// |
| 378 | |
| 379 | void KernelEnvironmentOp::getSuccessorRegions( |
| 380 | RegionBranchPoint point, SmallVectorImpl<RegionSuccessor> ®ions) { |
| 381 | getSingleRegionOpSuccessorRegions(op: getOperation(), region&: getRegion(), point, |
| 382 | regions); |
| 383 | } |
| 384 | |
| 385 | ValueRange KernelEnvironmentOp::getSuccessorInputs(RegionSuccessor successor) { |
| 386 | return getSingleRegionSuccessorInputs(op: getOperation(), successor); |
| 387 | } |
| 388 | |
| 389 | void KernelEnvironmentOp::getCanonicalizationPatterns( |
| 390 | RewritePatternSet &results, MLIRContext *context) { |
| 391 | results.add<RemoveEmptyKernelEnvironment, KernelEnvironmentDedupDataOperands>( |
| 392 | arg&: context); |
| 393 | } |
| 394 | |
| 395 | /// Extract async for `clauseDeviceType`. Returns true if a clause was found. |
| 396 | template <typename ComputeConstructT> |
| 397 | static bool |
| 398 | (ComputeConstructT computeConstruct, |
| 399 | DeviceType clauseDeviceType, MLIRContext *context, |
| 400 | std::optional<Value> &asyncOperand, UnitAttr &asyncOnly) { |
| 401 | if (computeConstruct.hasAsyncOnly(clauseDeviceType)) { |
| 402 | asyncOnly = UnitAttr::get(context); |
| 403 | return true; |
| 404 | } |
| 405 | if (Value asyncValue = computeConstruct.getAsyncValue(clauseDeviceType)) { |
| 406 | asyncOperand = asyncValue; |
| 407 | return true; |
| 408 | } |
| 409 | return false; |
| 410 | } |
| 411 | |
| 412 | /// Extract wait for `clauseDeviceType`. Returns true if a clause was found. |
| 413 | template <typename ComputeConstructT> |
| 414 | static bool (ComputeConstructT computeConstruct, |
| 415 | DeviceType clauseDeviceType, MLIRContext *context, |
| 416 | std::optional<Value> &waitDevnum, |
| 417 | SmallVectorImpl<Value> &waitOperands, |
| 418 | UnitAttr &waitOnly) { |
| 419 | if (computeConstruct.hasWaitOnly(clauseDeviceType)) { |
| 420 | waitOnly = UnitAttr::get(context); |
| 421 | return true; |
| 422 | } |
| 423 | Value devnum = computeConstruct.getWaitDevnum(clauseDeviceType); |
| 424 | auto waitValues = computeConstruct.getWaitValues(clauseDeviceType); |
| 425 | if (!devnum && waitValues.empty()) |
| 426 | return false; |
| 427 | if (devnum) |
| 428 | waitDevnum = devnum; |
| 429 | waitOperands.append(waitValues.begin(), waitValues.end()); |
| 430 | return true; |
| 431 | } |
| 432 | |
| 433 | template <typename ComputeConstructT> |
| 434 | static void populateKernelEnvironmentAsyncWait( |
| 435 | ComputeConstructT computeConstruct, DeviceType deviceType, |
| 436 | std::optional<Value> &asyncOperand, UnitAttr &asyncOnly, |
| 437 | std::optional<Value> &waitDevnum, SmallVectorImpl<Value> &waitOperands, |
| 438 | UnitAttr &waitOnly) { |
| 439 | MLIRContext *context = computeConstruct->getContext(); |
| 440 | |
| 441 | // Prefer device_type-specific clauses, then default ones. |
| 442 | if (!extractAsyncClause(computeConstruct, deviceType, context, asyncOperand, |
| 443 | asyncOnly)) { |
| 444 | if (deviceType != DeviceType::None) |
| 445 | extractAsyncClause(computeConstruct, DeviceType::None, context, |
| 446 | asyncOperand, asyncOnly); |
| 447 | } |
| 448 | |
| 449 | if (!extractWaitClause(computeConstruct, deviceType, context, waitDevnum, |
| 450 | waitOperands, waitOnly)) { |
| 451 | if (deviceType != DeviceType::None) |
| 452 | extractWaitClause(computeConstruct, DeviceType::None, context, waitDevnum, |
| 453 | waitOperands, waitOnly); |
| 454 | } |
| 455 | } |
| 456 | |
| 457 | template <typename ComputeConstructT> |
| 458 | KernelEnvironmentOp |
| 459 | KernelEnvironmentOp::createAndPopulate(ComputeConstructT computeConstruct, |
| 460 | DeviceType deviceType, |
| 461 | OpBuilder &builder) { |
| 462 | std::optional<Value> asyncOperand; |
| 463 | UnitAttr asyncOnly = nullptr; |
| 464 | std::optional<Value> waitDevnum; |
| 465 | SmallVector<Value> waitOperands; |
| 466 | UnitAttr waitOnly = nullptr; |
| 467 | populateKernelEnvironmentAsyncWait(computeConstruct, deviceType, asyncOperand, |
| 468 | asyncOnly, waitDevnum, waitOperands, |
| 469 | waitOnly); |
| 470 | |
| 471 | auto kernelEnvironment = KernelEnvironmentOp::create( |
| 472 | builder, computeConstruct->getLoc(), |
| 473 | computeConstruct.getDataClauseOperands(), asyncOperand.value_or(u: Value()), |
| 474 | asyncOnly, waitDevnum.value_or(u: Value()), waitOperands, waitOnly); |
| 475 | Block &block = kernelEnvironment.getRegion().emplaceBlock(); |
| 476 | builder.setInsertionPointToStart(&block); |
| 477 | return kernelEnvironment; |
| 478 | } |
| 479 | |
| 480 | template KernelEnvironmentOp |
| 481 | KernelEnvironmentOp::createAndPopulate<ParallelOp>(ParallelOp, DeviceType, |
| 482 | OpBuilder &); |
| 483 | template KernelEnvironmentOp |
| 484 | KernelEnvironmentOp::createAndPopulate<KernelsOp>(KernelsOp, DeviceType, |
| 485 | OpBuilder &); |
| 486 | template KernelEnvironmentOp |
| 487 | KernelEnvironmentOp::createAndPopulate<SerialOp>(SerialOp, DeviceType, |
| 488 | OpBuilder &); |
| 489 | |
| 490 | LogicalResult KernelEnvironmentOp::verify() { |
| 491 | if (getAsyncOnly() && getAsyncOperand()) |
| 492 | return emitError(message: "async-only cannot appear with async operand" ); |
| 493 | if (getWaitOnly() && (!getWaitOperands().empty() || getWaitDevnum())) |
| 494 | return emitError(message: "wait-only cannot appear with wait operands or devnum" ); |
| 495 | if ((*this)->getParentOfType<KernelEnvironmentOp>()) |
| 496 | return emitError( |
| 497 | message: "must not be nested within another acc.kernel_environment" ); |
| 498 | return success(); |
| 499 | } |
| 500 | |
| 501 | //===----------------------------------------------------------------------===// |
| 502 | // FirstprivateMapInitialOp |
| 503 | //===----------------------------------------------------------------------===// |
| 504 | |
| 505 | LogicalResult FirstprivateMapInitialOp::verify() { |
| 506 | if (getDataClause() != acc::DataClause::acc_firstprivate) |
| 507 | return emitError(message: "data clause associated with firstprivate operation must " |
| 508 | "match its intent" ); |
| 509 | if (!getVar()) |
| 510 | return emitError(message: "must have var operand" ); |
| 511 | if (!mlir::isa<mlir::acc::PointerLikeType>(Val: getVar().getType()) && |
| 512 | !mlir::isa<mlir::acc::MappableType>(Val: getVar().getType())) |
| 513 | return emitError(message: "var must be mappable or pointer-like" ); |
| 514 | if (mlir::isa<mlir::acc::PointerLikeType>(Val: getVar().getType()) && |
| 515 | getVarType() == getVar().getType()) |
| 516 | return emitError(message: "varType must capture the element type of var" ); |
| 517 | if (getModifiers() != acc::DataClauseModifier::none) |
| 518 | return emitError(message: "no data clause modifiers are allowed" ); |
| 519 | return success(); |
| 520 | } |
| 521 | |
| 522 | void FirstprivateMapInitialOp::getEffects( |
| 523 | SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>> |
| 524 | &effects) { |
| 525 | effects.emplace_back(Args: MemoryEffects::Read::get(), |
| 526 | Args: acc::CurrentDeviceIdResource::get()); |
| 527 | addOperandEffect<MemoryEffects::Read>(effects, operand: getVarMutable()); |
| 528 | addResultEffect<MemoryEffects::Write>(effects, result: getAccVar()); |
| 529 | } |
| 530 | |
| 531 | //===----------------------------------------------------------------------===// |
| 532 | // ReductionInitOp |
| 533 | //===----------------------------------------------------------------------===// |
| 534 | |
| 535 | void ReductionInitOp::getSuccessorRegions( |
| 536 | RegionBranchPoint point, SmallVectorImpl<RegionSuccessor> ®ions) { |
| 537 | getSingleRegionOpSuccessorRegions(op: getOperation(), region&: getRegion(), point, |
| 538 | regions); |
| 539 | } |
| 540 | |
| 541 | void ReductionInitOp::getRegionInvocationBounds( |
| 542 | ArrayRef<Attribute> operands, |
| 543 | SmallVectorImpl<InvocationBounds> &invocationBounds) { |
| 544 | invocationBounds.emplace_back(Args: 1, Args: 1); |
| 545 | } |
| 546 | |
| 547 | ValueRange ReductionInitOp::getSuccessorInputs(RegionSuccessor successor) { |
| 548 | return getSingleRegionSuccessorInputs(op: getOperation(), successor); |
| 549 | } |
| 550 | |
| 551 | LogicalResult ReductionInitOp::verify() { |
| 552 | Block &block = getRegion().front(); |
| 553 | if (auto yieldOp = dyn_cast<acc::YieldOp>(Val: block.getTerminator())) { |
| 554 | if (yieldOp.getNumOperands() != 1) |
| 555 | return emitOpError( |
| 556 | message: "region must yield exactly one value (private storage)" ); |
| 557 | if (yieldOp.getOperand(i: 0).getType() != getVar().getType()) |
| 558 | return emitOpError(message: "yielded value type must match var type" ); |
| 559 | } |
| 560 | return success(); |
| 561 | } |
| 562 | |
| 563 | //===----------------------------------------------------------------------===// |
| 564 | // ReductionCombineRegionOp |
| 565 | //===----------------------------------------------------------------------===// |
| 566 | |
| 567 | void ReductionCombineRegionOp::getSuccessorRegions( |
| 568 | RegionBranchPoint point, SmallVectorImpl<RegionSuccessor> ®ions) { |
| 569 | getSingleRegionOpSuccessorRegions(op: getOperation(), region&: getRegion(), point, |
| 570 | regions); |
| 571 | } |
| 572 | |
| 573 | void ReductionCombineRegionOp::getRegionInvocationBounds( |
| 574 | ArrayRef<Attribute> operands, |
| 575 | SmallVectorImpl<InvocationBounds> &invocationBounds) { |
| 576 | invocationBounds.emplace_back(Args: 1, Args: 1); |
| 577 | } |
| 578 | |
| 579 | ValueRange |
| 580 | ReductionCombineRegionOp::getSuccessorInputs(RegionSuccessor successor) { |
| 581 | return getSingleRegionSuccessorInputs(op: getOperation(), successor); |
| 582 | } |
| 583 | |
| 584 | LogicalResult ReductionCombineRegionOp::verify() { |
| 585 | Block &block = getRegion().front(); |
| 586 | if (auto yieldOp = dyn_cast<acc::YieldOp>(Val: block.getTerminator())) { |
| 587 | if (yieldOp.getNumOperands() != 0) |
| 588 | return emitOpError(message: "region must be terminated by acc.yield with no " |
| 589 | "operands" ); |
| 590 | } |
| 591 | return success(); |
| 592 | } |
| 593 | |
| 594 | //===----------------------------------------------------------------------===// |
| 595 | // ReductionAccumulateOp |
| 596 | //===----------------------------------------------------------------------===// |
| 597 | |
| 598 | LogicalResult ReductionAccumulateOp::verify() { |
| 599 | Type valueType = getValue().getType(); |
| 600 | auto ptrLikeTy = cast<PointerLikeType>(Val: getMemref().getType()); |
| 601 | Type elementType = ptrLikeTy.getElementType(); |
| 602 | if (!elementType) |
| 603 | return emitOpError(message: "pointer-like destination must have an element type" ); |
| 604 | if (elementType != valueType) |
| 605 | return emitOpError(message: "pointer-like element type must match value type" ); |
| 606 | if (getParDims().getArray().empty()) |
| 607 | return emitOpError(message: "par_dims must specify at least one parallel dimension" ); |
| 608 | return success(); |
| 609 | } |
| 610 | |
| 611 | //===----------------------------------------------------------------------===// |
| 612 | // ReductionAccumulateArrayOp |
| 613 | //===----------------------------------------------------------------------===// |
| 614 | |
| 615 | LogicalResult ReductionAccumulateArrayOp::verify() { |
| 616 | if (getParDims().getArray().empty()) |
| 617 | return emitOpError(message: "par_dims must specify at least one parallel dimension" ); |
| 618 | return success(); |
| 619 | } |
| 620 | |
| 621 | //===----------------------------------------------------------------------===// |
| 622 | // ReductionCombineOp |
| 623 | //===----------------------------------------------------------------------===// |
| 624 | |
| 625 | void ReductionCombineOp::getEffects( |
| 626 | SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>> |
| 627 | &effects) { |
| 628 | effects.emplace_back(Args: MemoryEffects::Read::get(), Args: &getSrcMemrefMutable(), |
| 629 | Args: SideEffects::DefaultResource::get()); |
| 630 | effects.emplace_back(Args: MemoryEffects::Read::get(), Args: &getDestMemrefMutable(), |
| 631 | Args: SideEffects::DefaultResource::get()); |
| 632 | effects.emplace_back(Args: MemoryEffects::Write::get(), Args: &getDestMemrefMutable(), |
| 633 | Args: SideEffects::DefaultResource::get()); |
| 634 | } |
| 635 | |
| 636 | //===----------------------------------------------------------------------===// |
| 637 | // ComputeRegionOp |
| 638 | //===----------------------------------------------------------------------===// |
| 639 | |
| 640 | static ParWidthOp getParWidthOpForLaunchArg(ComputeRegionOp op, |
| 641 | GPUParallelDimAttr parDim) { |
| 642 | for (auto launchArg : op.getLaunchArgs()) { |
| 643 | auto parOp = launchArg.getDefiningOp<ParWidthOp>(); |
| 644 | if (!parOp) |
| 645 | continue; |
| 646 | auto launchArgDim = cast<GPUParallelDimAttr>(Val: parOp.getParDim()); |
| 647 | if (launchArgDim == parDim) |
| 648 | return parOp; |
| 649 | } |
| 650 | return nullptr; |
| 651 | } |
| 652 | |
| 653 | std::optional<Value> ComputeRegionOp::getLaunchArg(GPUParallelDimAttr parDim) { |
| 654 | if (auto parWidthOp = getParWidthOpForLaunchArg(op: *this, parDim)) |
| 655 | return parWidthOp.getResult(); |
| 656 | return {}; |
| 657 | } |
| 658 | |
| 659 | std::optional<Value> |
| 660 | ComputeRegionOp::getKnownLaunchArg(GPUParallelDimAttr parDim) { |
| 661 | if (auto parWidthOp = getParWidthOpForLaunchArg(op: *this, parDim)) |
| 662 | if (parWidthOp.getLaunchArg()) |
| 663 | return parWidthOp.getLaunchArg(); |
| 664 | return {}; |
| 665 | } |
| 666 | |
| 667 | std::optional<uint64_t> |
| 668 | ComputeRegionOp::getKnownConstantLaunchArg(GPUParallelDimAttr parDim) { |
| 669 | auto knownParWidth = getKnownLaunchArg(parDim); |
| 670 | if (!knownParWidth.has_value()) |
| 671 | return {}; |
| 672 | std::optional<int64_t> width = getConstantIntValue(ofr: knownParWidth.value()); |
| 673 | if (!width || *width < 0) |
| 674 | return {}; |
| 675 | return static_cast<uint64_t>(*width); |
| 676 | } |
| 677 | |
| 678 | BlockArgument ComputeRegionOp::appendInputArg(Value value) { |
| 679 | getInputArgsMutable().append(values: value); |
| 680 | return getBody()->addArgument(type: value.getType(), loc: getLoc()); |
| 681 | } |
| 682 | |
| 683 | std::optional<BlockArgument> |
| 684 | ComputeRegionOp::wireHoistedValueThroughIns(Value value) { |
| 685 | Region ®ion = getRegion(); |
| 686 | |
| 687 | auto useIsInRegion = [&](OpOperand &use) -> bool { |
| 688 | return region.isAncestor(other: use.getOwner()->getParentRegion()); |
| 689 | }; |
| 690 | |
| 691 | if (!areValuesDefinedAbove(values: ValueRange(value), limit&: region) || |
| 692 | !llvm::any_of(Range: value.getUses(), P: useIsInRegion)) |
| 693 | return std::nullopt; |
| 694 | |
| 695 | BlockArgument arg = appendInputArg(value); |
| 696 | replaceAllUsesInRegionWith(orig: value, replacement: arg, region); |
| 697 | return arg; |
| 698 | } |
| 699 | |
| 700 | bool ComputeRegionOp::isEffectivelySerial() { |
| 701 | auto *ctx = getContext(); |
| 702 | |
| 703 | if (getLaunchArg(parDim: GPUParallelDimAttr::seqDim(ctx))) |
| 704 | return true; |
| 705 | |
| 706 | auto checkDim = [&](GPUParallelDimAttr dim) -> bool { |
| 707 | auto val = getKnownConstantLaunchArg(parDim: dim); |
| 708 | return val && *val == 1; |
| 709 | }; |
| 710 | |
| 711 | return checkDim(GPUParallelDimAttr::threadXDim(ctx)) && |
| 712 | checkDim(GPUParallelDimAttr::threadYDim(ctx)) && |
| 713 | checkDim(GPUParallelDimAttr::threadZDim(ctx)) && |
| 714 | checkDim(GPUParallelDimAttr::blockXDim(ctx)) && |
| 715 | checkDim(GPUParallelDimAttr::blockYDim(ctx)) && |
| 716 | checkDim(GPUParallelDimAttr::blockZDim(ctx)); |
| 717 | } |
| 718 | |
| 719 | bool ComputeRegionOp::launchesMultipleBlocks() { |
| 720 | return llvm::any_of(Range: getLaunchParDims(), P: [&](GPUParallelDimAttr dim) { |
| 721 | if (!dim.isAnyBlock()) |
| 722 | return false; |
| 723 | std::optional<uint64_t> width = getKnownConstantLaunchArg(parDim: dim); |
| 724 | return !width || *width != 1; |
| 725 | }); |
| 726 | } |
| 727 | |
| 728 | BlockArgument ComputeRegionOp::parDimToWidth(GPUParallelDimAttr parDim) { |
| 729 | for (auto [pos, launchArg] : llvm::enumerate(First: getLaunchArgs())) { |
| 730 | auto parOp = launchArg.getDefiningOp<ParWidthOp>(); |
| 731 | assert(parOp); |
| 732 | auto launchArgDim = cast<GPUParallelDimAttr>(Val: parOp.getParDim()); |
| 733 | if (launchArgDim == parDim) { |
| 734 | assert(pos < getRegion().front().getNumArguments() && |
| 735 | "launch arg position out of range" ); |
| 736 | return getRegion().front().getArgument(i: pos); |
| 737 | } |
| 738 | } |
| 739 | llvm_unreachable("attempting to get unspecified parDim" ); |
| 740 | } |
| 741 | |
| 742 | SmallVector<GPUParallelDimAttr> ComputeRegionOp::getLaunchParDims() { |
| 743 | SmallVector<GPUParallelDimAttr> parDims; |
| 744 | for (auto launchArg : getLaunchArgs()) { |
| 745 | auto parOp = launchArg.getDefiningOp<ParWidthOp>(); |
| 746 | auto launchArgDim = cast<GPUParallelDimAttr>(Val: parOp.getParDim()); |
| 747 | int64_t dimInt = launchArgDim.getValue().getInt(); |
| 748 | parDims.push_back(Elt: intToParDim(context: getContext(), dimInt)); |
| 749 | } |
| 750 | return parDims; |
| 751 | } |
| 752 | |
| 753 | Value ComputeRegionOp::getOperand(BlockArgument blockArg) { |
| 754 | Block *body = getBody(); |
| 755 | if (blockArg.getOwner() != body) |
| 756 | return Value(); |
| 757 | unsigned argNumber = blockArg.getArgNumber(); |
| 758 | unsigned numLaunchArgs = getLaunchArgs().size(); |
| 759 | unsigned numInputArgs = getInputArgs().size(); |
| 760 | if (argNumber >= numLaunchArgs + numInputArgs) |
| 761 | return Value(); |
| 762 | if (argNumber < numLaunchArgs) |
| 763 | return getLaunchArgs()[argNumber]; |
| 764 | return getInputArgs()[argNumber - numLaunchArgs]; |
| 765 | } |
| 766 | |
| 767 | std::optional<BlockArgument> ComputeRegionOp::getBlockArg(Value value) { |
| 768 | Block *body = getBody(); |
| 769 | for (auto [idx, launchVal] : llvm::enumerate(First: getLaunchArgs())) { |
| 770 | if (launchVal == value) |
| 771 | return body->getArgument(i: idx); |
| 772 | } |
| 773 | unsigned numLaunch = getLaunchArgs().size(); |
| 774 | for (auto [idx, inputVal] : llvm::enumerate(First: getInputArgs())) { |
| 775 | if (inputVal == value) |
| 776 | return body->getArgument(i: numLaunch + idx); |
| 777 | } |
| 778 | return std::nullopt; |
| 779 | } |
| 780 | |
| 781 | void ComputeRegionOp::getCanonicalizationPatterns(RewritePatternSet &results, |
| 782 | MLIRContext *context) { |
| 783 | results.add<ComputeRegionRemoveDuplicateArgs, ComputeRegionRemoveUnusedArgs>( |
| 784 | arg&: context); |
| 785 | } |
| 786 | |
| 787 | BlockArgument ComputeRegionOp::gpuParWidth(gpu::Processor processor) { |
| 788 | return parDimToWidth(parDim: GPUParallelDimAttr::get(getContext(), processor)); |
| 789 | } |
| 790 | |
| 791 | LogicalResult ComputeRegionOp::verify() { |
| 792 | for (auto op : getLaunchArgs()) |
| 793 | if (!op.getDefiningOp<acc::ParWidthOp>()) |
| 794 | return emitOpError( |
| 795 | message: "launch arguments must be results of acc.par_width operations" ); |
| 796 | |
| 797 | unsigned expectedBlockArgs = getLaunchArgs().size() + getInputArgs().size(); |
| 798 | unsigned actualBlockArgs = getRegion().front().getNumArguments(); |
| 799 | if (expectedBlockArgs != actualBlockArgs) |
| 800 | return emitOpError(message: "expected " ) |
| 801 | << expectedBlockArgs << " block arguments (launch + input), got " |
| 802 | << actualBlockArgs; |
| 803 | |
| 804 | return success(); |
| 805 | } |
| 806 | |
| 807 | void ComputeRegionOp::print(OpAsmPrinter &p) { |
| 808 | ValueRange regionArgs = getBody()->getArguments(); |
| 809 | ValueRange launchArgs = getLaunchArgs(); |
| 810 | ValueRange inputArgs = getInputArgs(); |
| 811 | |
| 812 | assert(regionArgs.size() == (launchArgs.size() + inputArgs.size()) && |
| 813 | "region args mismatch" ); |
| 814 | |
| 815 | if (getStream()) |
| 816 | p << " stream(" << getStream() << " : " << getStream().getType() << ")" ; |
| 817 | |
| 818 | size_t i = 0; |
| 819 | if (!launchArgs.empty()) { |
| 820 | p << " launch(" ; |
| 821 | for (size_t j = 0; j < launchArgs.size(); ++j, ++i) { |
| 822 | p << regionArgs[i] << " = " << launchArgs[j]; |
| 823 | if (j < launchArgs.size() - 1) |
| 824 | p << ", " ; |
| 825 | } |
| 826 | p << ")" ; |
| 827 | } |
| 828 | if (!inputArgs.empty()) { |
| 829 | p << " ins(" ; |
| 830 | for (size_t j = 0; j < inputArgs.size(); ++j, ++i) { |
| 831 | p << regionArgs[i] << " = " << inputArgs[j]; |
| 832 | if (j < inputArgs.size() - 1) |
| 833 | p << ", " ; |
| 834 | } |
| 835 | p << ") : (" ; |
| 836 | for (size_t j = 0; j < inputArgs.size(); ++j) { |
| 837 | p << inputArgs[j].getType(); |
| 838 | if (j < inputArgs.size() - 1) |
| 839 | p << ", " ; |
| 840 | } |
| 841 | p << ")" ; |
| 842 | } |
| 843 | p.printOptionalArrowTypeList(types: getResultTypes()); |
| 844 | p << " " ; |
| 845 | p.printRegion(blocks&: getRegion(), /*printEntryBlockArgs=*/false); |
| 846 | ComputeRegionOp::printProperties(ctx: getContext(), p, properties: getProperties(), |
| 847 | /*elidedProps=*/getOperandSegmentSizeAttr()); |
| 848 | p.printOptionalAttrDict(attrs: (*this)->getDiscardableAttrDictionary().getValue()); |
| 849 | } |
| 850 | |
| 851 | ParseResult ComputeRegionOp::parse(OpAsmParser &parser, |
| 852 | OperationState &result) { |
| 853 | auto &builder = parser.getBuilder(); |
| 854 | |
| 855 | SmallVector<OpAsmParser::Argument> regionArgs; |
| 856 | OpAsmParser::UnresolvedOperand streamOperand; |
| 857 | Type streamType; |
| 858 | SmallVector<OpAsmParser::UnresolvedOperand> launchOperands; |
| 859 | SmallVector<OpAsmParser::UnresolvedOperand> inputOperands; |
| 860 | SmallVector<Type> types; |
| 861 | |
| 862 | bool hasStream = false; |
| 863 | if (succeeded(Result: parser.parseOptionalKeyword(keyword: "stream" ))) { |
| 864 | hasStream = true; |
| 865 | if (parser.parseLParen() || parser.parseOperand(result&: streamOperand) || |
| 866 | parser.parseColon() || parser.parseType(result&: streamType) || |
| 867 | parser.parseRParen()) |
| 868 | return failure(); |
| 869 | } |
| 870 | |
| 871 | if (succeeded(Result: parser.parseOptionalKeyword(keyword: "launch" ))) { |
| 872 | if (parser.parseAssignmentList(lhs&: regionArgs, rhs&: launchOperands)) |
| 873 | return failure(); |
| 874 | Type indexType = builder.getIndexType(); |
| 875 | for (size_t i = 0; i < regionArgs.size(); ++i) |
| 876 | types.push_back(Elt: indexType); |
| 877 | } |
| 878 | |
| 879 | if (succeeded(Result: parser.parseOptionalKeyword(keyword: "ins" ))) { |
| 880 | if (parser.parseAssignmentList(lhs&: regionArgs, rhs&: inputOperands) || |
| 881 | parser.parseColon() || parser.parseLParen() || |
| 882 | parser.parseTypeList(result&: types) || parser.parseRParen()) |
| 883 | return failure(); |
| 884 | } |
| 885 | |
| 886 | if (parser.parseOptionalArrowTypeList(result&: result.types)) |
| 887 | return failure(); |
| 888 | |
| 889 | for (auto [iterArg, type] : llvm::zip_equal(t&: regionArgs, u&: types)) |
| 890 | iterArg.type = type; |
| 891 | |
| 892 | Region *body = result.addRegion(); |
| 893 | if (parser.parseRegion(region&: *body, arguments: regionArgs)) |
| 894 | return failure(); |
| 895 | ComputeRegionOp::ensureTerminator(region&: *body, builder&: parser.getBuilder(), |
| 896 | loc: result.location); |
| 897 | |
| 898 | const size_t numLaunchOperands = launchOperands.size(); |
| 899 | const size_t numInputOperands = inputOperands.size(); |
| 900 | assert(numLaunchOperands + numInputOperands == regionArgs.size() && |
| 901 | "compute region args mismatch" ); |
| 902 | |
| 903 | DenseI32ArrayAttr operandSegmentSizes = builder.getDenseI32ArrayAttr( |
| 904 | values: {static_cast<int32_t>(numLaunchOperands), |
| 905 | static_cast<int32_t>(numInputOperands), hasStream ? 1 : 0}); |
| 906 | |
| 907 | for (size_t i = 0; i < numLaunchOperands; ++i) { |
| 908 | if (parser.resolveOperand(operand: launchOperands[i], type: types[i], result&: result.operands)) |
| 909 | return failure(); |
| 910 | } |
| 911 | |
| 912 | for (size_t i = numLaunchOperands; i < regionArgs.size(); ++i) { |
| 913 | if (parser.resolveOperand(operand: inputOperands[i - numLaunchOperands], type: types[i], |
| 914 | result&: result.operands)) |
| 915 | return failure(); |
| 916 | } |
| 917 | |
| 918 | if (hasStream) { |
| 919 | if (parser.resolveOperand(operand: streamOperand, type: streamType, result&: result.operands)) |
| 920 | return failure(); |
| 921 | } |
| 922 | |
| 923 | Attribute parsedProperties; |
| 924 | if (ComputeRegionOp::genericParseProperties(parser, result&: parsedProperties)) |
| 925 | return failure(); |
| 926 | auto propertyDictionary = dyn_cast_or_null<DictionaryAttr>(Val&: parsedProperties); |
| 927 | if (parsedProperties && !propertyDictionary) |
| 928 | return parser.emitError(loc: parser.getNameLoc(), |
| 929 | message: "expected properties dictionary" ); |
| 930 | |
| 931 | NamedAttrList properties(propertyDictionary ? propertyDictionary |
| 932 | : builder.getDictionaryAttr(value: {})); |
| 933 | properties.set(name: ComputeRegionOp::getOperandSegmentSizeAttr(), |
| 934 | value: operandSegmentSizes); |
| 935 | propertyDictionary = properties.getDictionary(context: builder.getContext()); |
| 936 | auto emitError = [&]() { |
| 937 | return mlir::emitError(loc: result.location, message: "invalid properties " ) |
| 938 | << propertyDictionary << " for op " << result.name.getStringRef() |
| 939 | << ": " ; |
| 940 | }; |
| 941 | if (failed(Result: ComputeRegionOp::setPropertiesFromParsedAttr( |
| 942 | prop&: result.getOrAddProperties<Properties>(), attr: propertyDictionary, |
| 943 | emitError))) |
| 944 | return failure(); |
| 945 | |
| 946 | auto attrsLoc = parser.getCurrentLocation(); |
| 947 | if (parser.parseOptionalAttrDict(result&: result.attributes)) |
| 948 | return failure(); |
| 949 | for (StringRef attrName : ComputeRegionOp::getAttributeNames()) { |
| 950 | if (result.attributes.get(name: attrName)) |
| 951 | return parser.emitError(loc: attrsLoc) |
| 952 | << "inherent attribute '" << attrName |
| 953 | << "' cannot be parsed from attr-dict when strict properties in " |
| 954 | "assembly format is enabled" ; |
| 955 | } |
| 956 | |
| 957 | return success(); |
| 958 | } |
| 959 | |
| 960 | //===----------------------------------------------------------------------===// |
| 961 | // GPUSharedMemoryOp |
| 962 | //===----------------------------------------------------------------------===// |
| 963 | |
| 964 | LogicalResult GPUSharedMemoryOp::verify() { |
| 965 | if (getNumCopies() <= 0) |
| 966 | return emitOpError(message: "num_copies must be positive" ); |
| 967 | if (getStaticUpperBoundBytes() <= 0) |
| 968 | return emitOpError(message: "static_upper_bound_bytes must be positive" ); |
| 969 | |
| 970 | bool hasScaling = static_cast<bool>(getDynamicSharedMemoryScalingBytes()); |
| 971 | bool hasFixed = static_cast<bool>(getDynamicSharedMemoryFixedBytes()); |
| 972 | if (hasScaling != hasFixed) |
| 973 | return emitOpError( |
| 974 | message: "dynamic_shared_memory_scaling_bytes and " |
| 975 | "dynamic_shared_memory_fixed_bytes must both be present or both be " |
| 976 | "absent" ); |
| 977 | if (auto scalingAttr = getDynamicSharedMemoryScalingBytesAttr()) |
| 978 | if (scalingAttr.getValue().isNegative()) |
| 979 | return emitOpError(message: "dynamic_shared_memory_scaling_bytes must be " |
| 980 | "non-negative" ); |
| 981 | if (auto fixedAttr = getDynamicSharedMemoryFixedBytesAttr()) |
| 982 | if (fixedAttr.getValue().isNegative()) |
| 983 | return emitOpError(message: "dynamic_shared_memory_fixed_bytes must be " |
| 984 | "non-negative" ); |
| 985 | |
| 986 | auto resultTy = cast<MemRefType>(Val: getResult().getType()); |
| 987 | auto addrSpace = |
| 988 | dyn_cast_if_present<gpu::AddressSpaceAttr>(Val: resultTy.getMemorySpace()); |
| 989 | if (!addrSpace || addrSpace.getValue() != gpu::AddressSpace::Workgroup) |
| 990 | return emitOpError(message: "result memref must use #gpu.address_space<workgroup>" ); |
| 991 | |
| 992 | return success(); |
| 993 | } |
| 994 | |
| 995 | //===----------------------------------------------------------------------===// |
| 996 | // PredicateRegionOp |
| 997 | //===----------------------------------------------------------------------===// |
| 998 | |
| 999 | LogicalResult PredicateRegionOp::verify() { |
| 1000 | if (getRegion().empty()) |
| 1001 | return emitOpError(message: "region needs to have at least one block" ); |
| 1002 | if (getRegion().front().getNumArguments() > 0) |
| 1003 | return emitOpError(message: "region cannot have any arguments" ); |
| 1004 | if (!getOperation()->getParentOfType<ComputeRegionOp>()) |
| 1005 | return emitOpError(message: "must be nested within an acc.compute_region operation" ); |
| 1006 | return success(); |
| 1007 | } |
| 1008 | |
| 1009 | //===----------------------------------------------------------------------===// |
| 1010 | // MapInfoOp |
| 1011 | //===----------------------------------------------------------------------===// |
| 1012 | |
| 1013 | LogicalResult MapInfoOp::verify() { |
| 1014 | // A pointer-like var addresses the mapped object, so varType has to name that |
| 1015 | // object rather than the address of it. |
| 1016 | if (mlir::isa<acc::PointerLikeType>(Val: getVar().getType()) && |
| 1017 | getVarType() == getVar().getType()) |
| 1018 | return emitOpError(message: "varType must capture the element type of var" ); |
| 1019 | |
| 1020 | // A descriptor operand is only meaningful together with the layout it |
| 1021 | // follows. Without a kind, consumers have no way to interpret it. |
| 1022 | if (getDesc() && getDescKind() == DataDescKind::none) |
| 1023 | return emitOpError(message: "desc requires a descKind other than none" ); |
| 1024 | |
| 1025 | // Bounds count elements of the mapped object, which is the OpenACC |
| 1026 | // descriptor layout, so it must be among the kinds named here. |
| 1027 | if (!getBounds().empty() && |
| 1028 | !acc::bitEnumContainsAny(bits: getDescKind(), bit: DataDescKind::openacc)) |
| 1029 | return emitOpError(message: "bounds require descKind openacc" ); |
| 1030 | |
| 1031 | // Anything below -1 has no meaning: -1 states that the size is unknown at |
| 1032 | // compile time and 0 defers it to bounds or to a descriptor. |
| 1033 | if (getSize()) { |
| 1034 | std::optional<int64_t> constantSize = getConstantIntValue(ofr: getSize()); |
| 1035 | if (constantSize && *constantSize < -1) |
| 1036 | return emitOpError(message: "size must be -1, 0, or a positive byte count" ); |
| 1037 | } |
| 1038 | |
| 1039 | return success(); |
| 1040 | } |
| 1041 | |
| 1042 | //===----------------------------------------------------------------------===// |
| 1043 | // GPUParallelDimAttr |
| 1044 | //===----------------------------------------------------------------------===// |
| 1045 | |
| 1046 | GPUParallelDimAttr GPUParallelDimAttr::get(MLIRContext *context, |
| 1047 | gpu::Processor proc) { |
| 1048 | return processorParDim(context, proc); |
| 1049 | } |
| 1050 | |
| 1051 | GPUParallelDimAttr GPUParallelDimAttr::seqDim(MLIRContext *context) { |
| 1052 | return processorParDim(context, proc: gpu::Processor::Sequential); |
| 1053 | } |
| 1054 | |
| 1055 | GPUParallelDimAttr GPUParallelDimAttr::threadXDim(MLIRContext *context) { |
| 1056 | return processorParDim(context, proc: gpu::Processor::ThreadX); |
| 1057 | } |
| 1058 | |
| 1059 | GPUParallelDimAttr GPUParallelDimAttr::threadYDim(MLIRContext *context) { |
| 1060 | return processorParDim(context, proc: gpu::Processor::ThreadY); |
| 1061 | } |
| 1062 | |
| 1063 | GPUParallelDimAttr GPUParallelDimAttr::threadZDim(MLIRContext *context) { |
| 1064 | return processorParDim(context, proc: gpu::Processor::ThreadZ); |
| 1065 | } |
| 1066 | |
| 1067 | GPUParallelDimAttr GPUParallelDimAttr::blockXDim(MLIRContext *context) { |
| 1068 | return processorParDim(context, proc: gpu::Processor::BlockX); |
| 1069 | } |
| 1070 | |
| 1071 | GPUParallelDimAttr GPUParallelDimAttr::blockYDim(MLIRContext *context) { |
| 1072 | return processorParDim(context, proc: gpu::Processor::BlockY); |
| 1073 | } |
| 1074 | |
| 1075 | GPUParallelDimAttr GPUParallelDimAttr::blockZDim(MLIRContext *context) { |
| 1076 | return processorParDim(context, proc: gpu::Processor::BlockZ); |
| 1077 | } |
| 1078 | |
| 1079 | Attribute GPUParallelDimAttr::parse(AsmParser &parser, Type type) { |
| 1080 | GPUParallelDimAttr dim; |
| 1081 | if (parser.parseLess() || parseProcessorValue(parser, dim) || |
| 1082 | parser.parseGreater()) { |
| 1083 | parser.emitError(loc: parser.getCurrentLocation(), |
| 1084 | message: "expected format `<` processor_name `>`" ); |
| 1085 | return {}; |
| 1086 | } |
| 1087 | return dim; |
| 1088 | } |
| 1089 | |
| 1090 | void GPUParallelDimAttr::print(AsmPrinter &printer) const { |
| 1091 | printer << "<" ; |
| 1092 | printProcessorValue(printer, attr: *this); |
| 1093 | printer << ">" ; |
| 1094 | } |
| 1095 | |
| 1096 | GPUParallelDimAttr GPUParallelDimAttr::threadDim(MLIRContext *context, |
| 1097 | unsigned index) { |
| 1098 | assert(index <= 2 && "thread dimension index must be 0, 1, or 2" ); |
| 1099 | switch (index) { |
| 1100 | case 0: |
| 1101 | return threadXDim(context); |
| 1102 | case 1: |
| 1103 | return threadYDim(context); |
| 1104 | case 2: |
| 1105 | return threadZDim(context); |
| 1106 | } |
| 1107 | llvm_unreachable("validated thread dimension index" ); |
| 1108 | } |
| 1109 | |
| 1110 | GPUParallelDimAttr GPUParallelDimAttr::blockDim(MLIRContext *context, |
| 1111 | unsigned index) { |
| 1112 | assert(index <= 2 && "block dimension index must be 0, 1, or 2" ); |
| 1113 | switch (index) { |
| 1114 | case 0: |
| 1115 | return blockXDim(context); |
| 1116 | case 1: |
| 1117 | return blockYDim(context); |
| 1118 | case 2: |
| 1119 | return blockZDim(context); |
| 1120 | } |
| 1121 | llvm_unreachable("validated block dimension index" ); |
| 1122 | } |
| 1123 | |
| 1124 | gpu::Processor GPUParallelDimAttr::getProcessor() const { |
| 1125 | return indexToGpuProcessor(idx: getValue().getInt()); |
| 1126 | } |
| 1127 | |
| 1128 | int GPUParallelDimAttr::getOrder() const { |
| 1129 | return gpuProcessorIndex(p: getProcessor()); |
| 1130 | } |
| 1131 | |
| 1132 | GPUParallelDimAttr GPUParallelDimAttr::getOneHigher() const { |
| 1133 | int order = getOrder(); |
| 1134 | if (order >= 6) // BlockZ is the highest |
| 1135 | return *this; |
| 1136 | return get(context: getContext(), proc: indexToGpuProcessor(idx: order + 1)); |
| 1137 | } |
| 1138 | |
| 1139 | GPUParallelDimAttr GPUParallelDimAttr::getOneLower() const { |
| 1140 | int order = getOrder(); |
| 1141 | if (order <= 0) // Sequential is the lowest |
| 1142 | return *this; |
| 1143 | return get(context: getContext(), proc: indexToGpuProcessor(idx: order - 1)); |
| 1144 | } |
| 1145 | |
| 1146 | bool GPUParallelDimAttr::isSeq() const { |
| 1147 | return getProcessor() == gpu::Processor::Sequential; |
| 1148 | } |
| 1149 | bool GPUParallelDimAttr::isThreadX() const { |
| 1150 | return getProcessor() == gpu::Processor::ThreadX; |
| 1151 | } |
| 1152 | bool GPUParallelDimAttr::isThreadY() const { |
| 1153 | return getProcessor() == gpu::Processor::ThreadY; |
| 1154 | } |
| 1155 | bool GPUParallelDimAttr::isThreadZ() const { |
| 1156 | return getProcessor() == gpu::Processor::ThreadZ; |
| 1157 | } |
| 1158 | bool GPUParallelDimAttr::isBlockX() const { |
| 1159 | return getProcessor() == gpu::Processor::BlockX; |
| 1160 | } |
| 1161 | bool GPUParallelDimAttr::isBlockY() const { |
| 1162 | return getProcessor() == gpu::Processor::BlockY; |
| 1163 | } |
| 1164 | bool GPUParallelDimAttr::isBlockZ() const { |
| 1165 | return getProcessor() == gpu::Processor::BlockZ; |
| 1166 | } |
| 1167 | bool GPUParallelDimAttr::isAnyThread() const { |
| 1168 | return isThreadX() || isThreadY() || isThreadZ(); |
| 1169 | } |
| 1170 | bool GPUParallelDimAttr::isAnyBlock() const { |
| 1171 | return isBlockX() || isBlockY() || isBlockZ(); |
| 1172 | } |
| 1173 | |
| 1174 | //===----------------------------------------------------------------------===// |
| 1175 | // GPUParallelDimsAttr |
| 1176 | //===----------------------------------------------------------------------===// |
| 1177 | |
| 1178 | GPUParallelDimsAttr GPUParallelDimsAttr::seq(MLIRContext *ctx) { |
| 1179 | return GPUParallelDimsAttr::get(context: ctx, array: {GPUParallelDimAttr::seqDim(context: ctx)}); |
| 1180 | } |
| 1181 | |
| 1182 | bool GPUParallelDimsAttr::isSeq() const { |
| 1183 | assert(!getArray().empty() && "no par_dims found" ); |
| 1184 | if (getArray().size() == 1) { |
| 1185 | auto parDim = dyn_cast<GPUParallelDimAttr>(Val: getArray()[0]); |
| 1186 | assert(parDim && "expected GPUParallelDimAttr" ); |
| 1187 | return parDim.isSeq(); |
| 1188 | } |
| 1189 | return false; |
| 1190 | } |
| 1191 | |
| 1192 | bool GPUParallelDimsAttr::isParallel() const { return !isSeq(); } |
| 1193 | |
| 1194 | bool GPUParallelDimsAttr::isMultiDim() const { return getArray().size() > 1; } |
| 1195 | |
| 1196 | bool GPUParallelDimsAttr::hasAnyBlockLevel() const { |
| 1197 | return llvm::any_of( |
| 1198 | Range: getArray(), P: [](const GPUParallelDimAttr &p) { return p.isAnyBlock(); }); |
| 1199 | } |
| 1200 | |
| 1201 | bool GPUParallelDimsAttr::hasOnlyBlockLevel() const { |
| 1202 | return !getArray().empty() && |
| 1203 | llvm::all_of(Range: getArray(), P: [](const GPUParallelDimAttr &p) { |
| 1204 | return p.isAnyBlock(); |
| 1205 | }); |
| 1206 | } |
| 1207 | |
| 1208 | bool GPUParallelDimsAttr::hasOnlyThreadYLevel() const { |
| 1209 | return !getArray().empty() && |
| 1210 | llvm::all_of(Range: getArray(), P: [](const GPUParallelDimAttr &p) { |
| 1211 | return p.isThreadY(); |
| 1212 | }); |
| 1213 | } |
| 1214 | |
| 1215 | bool GPUParallelDimsAttr::hasOnlyThreadXLevel() const { |
| 1216 | return !getArray().empty() && |
| 1217 | llvm::all_of(Range: getArray(), P: [](const GPUParallelDimAttr &p) { |
| 1218 | return p.isThreadX(); |
| 1219 | }); |
| 1220 | } |
| 1221 | |
| 1222 | Attribute GPUParallelDimsAttr::parse(AsmParser &parser, Type type) { |
| 1223 | FailureOr<SmallVector<GPUParallelDimAttr>> parDims = |
| 1224 | parseGPUParallelDimList(parser); |
| 1225 | if (failed(Result: parDims)) |
| 1226 | return {}; |
| 1227 | return GPUParallelDimsAttr::get(context: parser.getContext(), array: *parDims); |
| 1228 | } |
| 1229 | |
| 1230 | void GPUParallelDimsAttr::print(AsmPrinter &printer) const { |
| 1231 | printGPUParallelDimList(printer, dims: getArray()); |
| 1232 | } |
| 1233 | |
| 1234 | //===----------------------------------------------------------------------===// |
| 1235 | // ActiveParDimsAttr |
| 1236 | //===----------------------------------------------------------------------===// |
| 1237 | |
| 1238 | Attribute ActiveParDimsAttr::parse(AsmParser &parser, Type type) { |
| 1239 | FailureOr<SmallVector<GPUParallelDimAttr>> parDims = |
| 1240 | parseGPUParallelDimList(parser); |
| 1241 | if (failed(Result: parDims)) |
| 1242 | return {}; |
| 1243 | return ActiveParDimsAttr::get(context: parser.getContext(), array: *parDims); |
| 1244 | } |
| 1245 | |
| 1246 | void ActiveParDimsAttr::print(AsmPrinter &printer) const { |
| 1247 | printGPUParallelDimList(printer, dims: getArray()); |
| 1248 | } |
| 1249 | |