| 1 | //===- GPUBase.cpp - GPU types and attributes -----------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file implements GPU types, attributes, and enums. |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #include "mlir/Dialect/GPU/IR/GPUBase.h" |
| 14 | |
| 15 | #include "mlir/IR/Builders.h" |
| 16 | #include "mlir/IR/DialectImplementation.h" |
| 17 | #include "llvm/ADT/TypeSwitch.h" |
| 18 | #include "llvm/Support/CommandLine.h" |
| 19 | #include "llvm/Support/StringSaver.h" |
| 20 | |
| 21 | using namespace mlir; |
| 22 | using namespace mlir::gpu; |
| 23 | |
| 24 | //===----------------------------------------------------------------------===// |
| 25 | // GPU Device Mapping Attributes |
| 26 | //===----------------------------------------------------------------------===// |
| 27 | |
| 28 | int64_t GPUBlockMappingAttr::getMappingId() const { |
| 29 | return static_cast<int64_t>(getBlock()); |
| 30 | } |
| 31 | |
| 32 | bool GPUBlockMappingAttr::isLinearMapping() const { |
| 33 | return getMappingId() >= static_cast<int64_t>(MappingId::LinearDim0); |
| 34 | } |
| 35 | |
| 36 | int64_t GPUBlockMappingAttr::getRelativeIndex() const { |
| 37 | return isLinearMapping() |
| 38 | ? getMappingId() - static_cast<int64_t>(MappingId::LinearDim0) |
| 39 | : getMappingId(); |
| 40 | } |
| 41 | |
| 42 | int64_t GPUWarpgroupMappingAttr::getMappingId() const { |
| 43 | return static_cast<int64_t>(getWarpgroup()); |
| 44 | } |
| 45 | |
| 46 | bool GPUWarpgroupMappingAttr::isLinearMapping() const { |
| 47 | return getMappingId() >= static_cast<int64_t>(MappingId::LinearDim0); |
| 48 | } |
| 49 | |
| 50 | int64_t GPUWarpgroupMappingAttr::getRelativeIndex() const { |
| 51 | return isLinearMapping() |
| 52 | ? getMappingId() - static_cast<int64_t>(MappingId::LinearDim0) |
| 53 | : getMappingId(); |
| 54 | } |
| 55 | |
| 56 | int64_t GPUWarpMappingAttr::getMappingId() const { |
| 57 | return static_cast<int64_t>(getWarp()); |
| 58 | } |
| 59 | |
| 60 | bool GPUWarpMappingAttr::isLinearMapping() const { |
| 61 | return getMappingId() >= static_cast<int64_t>(MappingId::LinearDim0); |
| 62 | } |
| 63 | |
| 64 | int64_t GPUWarpMappingAttr::getRelativeIndex() const { |
| 65 | return isLinearMapping() |
| 66 | ? getMappingId() - static_cast<int64_t>(MappingId::LinearDim0) |
| 67 | : getMappingId(); |
| 68 | } |
| 69 | |
| 70 | int64_t GPUThreadMappingAttr::getMappingId() const { |
| 71 | return static_cast<int64_t>(getThread()); |
| 72 | } |
| 73 | |
| 74 | bool GPUThreadMappingAttr::isLinearMapping() const { |
| 75 | return getMappingId() >= static_cast<int64_t>(MappingId::LinearDim0); |
| 76 | } |
| 77 | |
| 78 | int64_t GPUThreadMappingAttr::getRelativeIndex() const { |
| 79 | return isLinearMapping() |
| 80 | ? getMappingId() - static_cast<int64_t>(MappingId::LinearDim0) |
| 81 | : getMappingId(); |
| 82 | } |
| 83 | |
| 84 | int64_t GPULaneMappingAttr::getMappingId() const { |
| 85 | return static_cast<int64_t>(getLane()); |
| 86 | } |
| 87 | |
| 88 | bool GPULaneMappingAttr::isLinearMapping() const { |
| 89 | return getMappingId() >= static_cast<int64_t>(MappingId::LinearDim0); |
| 90 | } |
| 91 | |
| 92 | int64_t GPULaneMappingAttr::getRelativeIndex() const { |
| 93 | return isLinearMapping() |
| 94 | ? getMappingId() - static_cast<int64_t>(MappingId::LinearDim0) |
| 95 | : getMappingId(); |
| 96 | } |
| 97 | |
| 98 | int64_t GPUMemorySpaceMappingAttr::getMappingId() const { |
| 99 | return static_cast<int64_t>(getAddressSpace()); |
| 100 | } |
| 101 | |
| 102 | bool GPUMemorySpaceMappingAttr::isLinearMapping() const { |
| 103 | llvm_unreachable("GPUMemorySpaceMappingAttr does not support linear mapping" ); |
| 104 | } |
| 105 | |
| 106 | int64_t GPUMemorySpaceMappingAttr::getRelativeIndex() const { |
| 107 | llvm_unreachable("GPUMemorySpaceMappingAttr does not support relative index" ); |
| 108 | } |
| 109 | |
| 110 | //===----------------------------------------------------------------------===// |
| 111 | // MMAMatrixType |
| 112 | //===----------------------------------------------------------------------===// |
| 113 | |
| 114 | MMAMatrixType MMAMatrixType::get(ArrayRef<int64_t> shape, Type elementType, |
| 115 | StringRef operand) { |
| 116 | return Base::get(ctx: elementType.getContext(), args&: shape, args&: elementType, args&: operand); |
| 117 | } |
| 118 | |
| 119 | MMAMatrixType |
| 120 | MMAMatrixType::getChecked(function_ref<InFlightDiagnostic()> emitError, |
| 121 | ArrayRef<int64_t> shape, Type elementType, |
| 122 | StringRef operand) { |
| 123 | return Base::getChecked(emitErrorFn: emitError, ctx: elementType.getContext(), args: shape, |
| 124 | args: elementType, args: operand); |
| 125 | } |
| 126 | |
| 127 | unsigned MMAMatrixType::getNumDims() const { return getImpl()->numDims; } |
| 128 | |
| 129 | ArrayRef<int64_t> MMAMatrixType::getShape() const { |
| 130 | return getImpl()->getShape(); |
| 131 | } |
| 132 | |
| 133 | Type MMAMatrixType::getElementType() const { return getImpl()->elementType; } |
| 134 | |
| 135 | StringRef MMAMatrixType::getOperand() const { return getImpl()->getOperand(); } |
| 136 | |
| 137 | bool MMAMatrixType::isValidElementType(Type elementType) { |
| 138 | return elementType.isF16() || elementType.isF32() || elementType.isF64() || |
| 139 | elementType.isUnsignedInteger(width: 8) || elementType.isSignedInteger(width: 8) || |
| 140 | elementType.isInteger(width: 32); |
| 141 | } |
| 142 | |
| 143 | LogicalResult |
| 144 | MMAMatrixType::verifyInvariants(function_ref<InFlightDiagnostic()> emitError, |
| 145 | ArrayRef<int64_t> shape, Type elementType, |
| 146 | StringRef operand) { |
| 147 | if (operand != "AOp" && operand != "BOp" && operand != "COp" ) |
| 148 | return emitError() << "operand expected to be one of AOp, BOp or COp" ; |
| 149 | |
| 150 | if (shape.size() != 2) |
| 151 | return emitError() << "MMAMatrixType must have exactly two dimensions" ; |
| 152 | |
| 153 | if (!MMAMatrixType::isValidElementType(elementType)) |
| 154 | return emitError() |
| 155 | << "MMAMatrixType elements must be SI8, UI8, I32, F16, F32, or F64" ; |
| 156 | |
| 157 | return success(); |
| 158 | } |
| 159 | |
| 160 | //===----------------------------------------------------------------------===// |
| 161 | // GPUDialect |
| 162 | //===----------------------------------------------------------------------===// |
| 163 | |
| 164 | bool GPUDialect::isWorkgroupMemoryAddressSpace(Attribute memorySpace) { |
| 165 | if (!memorySpace) |
| 166 | return false; |
| 167 | if (auto gpuAttr = llvm::dyn_cast<gpu::AddressSpaceAttr>(Val&: memorySpace)) |
| 168 | return gpuAttr.getValue() == getWorkgroupAddressSpace(); |
| 169 | return false; |
| 170 | } |
| 171 | |
| 172 | bool GPUDialect::hasWorkgroupMemoryAddressSpace(MemRefType type) { |
| 173 | Attribute memorySpace = type.getMemorySpace(); |
| 174 | return isWorkgroupMemoryAddressSpace(memorySpace); |
| 175 | } |
| 176 | |
| 177 | bool GPUDialect::isConstantMemoryAddressSpace(Attribute memorySpace) { |
| 178 | if (!memorySpace) |
| 179 | return false; |
| 180 | if (auto gpuAttr = llvm::dyn_cast<gpu::AddressSpaceAttr>(Val&: memorySpace)) |
| 181 | return gpuAttr.getValue() == getConstantAddressSpace(); |
| 182 | return false; |
| 183 | } |
| 184 | |
| 185 | bool GPUDialect::hasConstantMemoryAddressSpace(MemRefType type) { |
| 186 | Attribute memorySpace = type.getMemorySpace(); |
| 187 | return isConstantMemoryAddressSpace(memorySpace); |
| 188 | } |
| 189 | |
| 190 | //===----------------------------------------------------------------------===// |
| 191 | // GPU object attribute |
| 192 | //===----------------------------------------------------------------------===// |
| 193 | |
| 194 | LogicalResult ObjectAttr::verify(function_ref<InFlightDiagnostic()> emitError, |
| 195 | Attribute target, CompilationTarget format, |
| 196 | StringAttr object, DictionaryAttr properties, |
| 197 | KernelTableAttr kernels) { |
| 198 | if (!target) |
| 199 | return emitError() << "the target attribute cannot be null" ; |
| 200 | if (target.hasPromiseOrImplementsInterface<TargetAttrInterface>()) |
| 201 | return success(); |
| 202 | return emitError() << "the target attribute must implement or promise the " |
| 203 | "`gpu::TargetAttrInterface`" ; |
| 204 | } |
| 205 | |
| 206 | namespace { |
| 207 | ParseResult parseObject(AsmParser &odsParser, CompilationTarget &format, |
| 208 | StringAttr &object) { |
| 209 | std::optional<CompilationTarget> formatResult; |
| 210 | StringRef enumKeyword; |
| 211 | auto loc = odsParser.getCurrentLocation(); |
| 212 | if (failed(Result: odsParser.parseOptionalKeyword(keyword: &enumKeyword))) |
| 213 | formatResult = CompilationTarget::Fatbin; |
| 214 | if (!formatResult && |
| 215 | (formatResult = |
| 216 | gpu::symbolizeEnum<gpu::CompilationTarget>(str: enumKeyword)) && |
| 217 | odsParser.parseEqual()) |
| 218 | return odsParser.emitError(loc, message: "expected an equal sign" ); |
| 219 | if (!formatResult) |
| 220 | return odsParser.emitError(loc, message: "expected keyword for GPU object format" ); |
| 221 | FailureOr<StringAttr> objectResult = |
| 222 | FieldParser<StringAttr>::parse(parser&: odsParser); |
| 223 | if (failed(Result: objectResult)) |
| 224 | return odsParser.emitError(loc: odsParser.getCurrentLocation(), |
| 225 | message: "failed to parse GPU_ObjectAttr parameter " |
| 226 | "'object' which is to be a `StringAttr`" ); |
| 227 | format = *formatResult; |
| 228 | object = *objectResult; |
| 229 | return success(); |
| 230 | } |
| 231 | |
| 232 | void printObject(AsmPrinter &odsParser, CompilationTarget format, |
| 233 | StringAttr object) { |
| 234 | if (format != CompilationTarget::Fatbin) |
| 235 | odsParser << stringifyEnum(enumValue: format) << " = " ; |
| 236 | odsParser << object; |
| 237 | } |
| 238 | } // namespace |
| 239 | |
| 240 | //===----------------------------------------------------------------------===// |
| 241 | // GPU select object attribute |
| 242 | //===----------------------------------------------------------------------===// |
| 243 | |
| 244 | LogicalResult |
| 245 | gpu::SelectObjectAttr::verify(function_ref<InFlightDiagnostic()> emitError, |
| 246 | Attribute target) { |
| 247 | // Check `target`, it can be null, an integer attr or a GPU Target attribute. |
| 248 | if (target) { |
| 249 | if (auto intAttr = mlir::dyn_cast<IntegerAttr>(Val&: target)) { |
| 250 | if (intAttr.getInt() < 0) { |
| 251 | return emitError() << "the object index must be positive" ; |
| 252 | } |
| 253 | } else if (!target.hasPromiseOrImplementsInterface<TargetAttrInterface>()) { |
| 254 | return emitError() |
| 255 | << "the target attribute must be a GPU Target attribute" ; |
| 256 | } |
| 257 | } |
| 258 | return success(); |
| 259 | } |
| 260 | |
| 261 | //===----------------------------------------------------------------------===// |
| 262 | // GPU KernelMetadataAttr |
| 263 | //===----------------------------------------------------------------------===// |
| 264 | |
| 265 | KernelMetadataAttr KernelMetadataAttr::get(FunctionOpInterface kernel, |
| 266 | DictionaryAttr metadata) { |
| 267 | assert(kernel && "invalid kernel" ); |
| 268 | return get(name: kernel.getNameAttr(), functionType: kernel.getFunctionType(), |
| 269 | argAttrs: kernel.getAllArgAttrs(), metadata); |
| 270 | } |
| 271 | |
| 272 | KernelMetadataAttr |
| 273 | KernelMetadataAttr::getChecked(function_ref<InFlightDiagnostic()> emitError, |
| 274 | FunctionOpInterface kernel, |
| 275 | DictionaryAttr metadata) { |
| 276 | assert(kernel && "invalid kernel" ); |
| 277 | return getChecked(emitError, name: kernel.getNameAttr(), functionType: kernel.getFunctionType(), |
| 278 | argAttrs: kernel.getAllArgAttrs(), metadata); |
| 279 | } |
| 280 | |
| 281 | KernelMetadataAttr |
| 282 | KernelMetadataAttr::appendMetadata(ArrayRef<NamedAttribute> attrs) const { |
| 283 | if (attrs.empty()) |
| 284 | return *this; |
| 285 | NamedAttrList attrList; |
| 286 | if (DictionaryAttr dict = getMetadata()) |
| 287 | attrList.append(newAttributes&: dict); |
| 288 | attrList.append(newAttributes&: attrs); |
| 289 | return KernelMetadataAttr::get(name: getName(), functionType: getFunctionType(), argAttrs: getArgAttrs(), |
| 290 | metadata: attrList.getDictionary(context: getContext())); |
| 291 | } |
| 292 | |
| 293 | LogicalResult |
| 294 | KernelMetadataAttr::verify(function_ref<InFlightDiagnostic()> emitError, |
| 295 | StringAttr name, Type functionType, |
| 296 | ArrayAttr argAttrs, DictionaryAttr metadata) { |
| 297 | if (name.empty()) |
| 298 | return emitError() << "the kernel name can't be empty" ; |
| 299 | if (argAttrs) { |
| 300 | if (llvm::any_of(Range&: argAttrs, P: [](Attribute attr) { |
| 301 | return !llvm::isa<DictionaryAttr>(Val: attr); |
| 302 | })) |
| 303 | return emitError() |
| 304 | << "all attributes in the array must be a dictionary attribute" ; |
| 305 | } |
| 306 | return success(); |
| 307 | } |
| 308 | |
| 309 | //===----------------------------------------------------------------------===// |
| 310 | // GPU KernelTableAttr |
| 311 | //===----------------------------------------------------------------------===// |
| 312 | |
| 313 | KernelTableAttr KernelTableAttr::get(MLIRContext *context, |
| 314 | ArrayRef<KernelMetadataAttr> kernels, |
| 315 | bool isSorted) { |
| 316 | // Note that `is_sorted` is always only invoked once even with assertions ON. |
| 317 | assert((!isSorted || llvm::is_sorted(kernels)) && |
| 318 | "expected a sorted kernel array" ); |
| 319 | // Immediately return the attribute if the array is sorted. |
| 320 | if (isSorted || llvm::is_sorted(Range&: kernels)) |
| 321 | return Base::get(ctx: context, args&: kernels); |
| 322 | // Sort the array. |
| 323 | SmallVector<KernelMetadataAttr> kernelsTmp(kernels); |
| 324 | llvm::array_pod_sort(Start: kernelsTmp.begin(), End: kernelsTmp.end()); |
| 325 | return Base::get(ctx: context, args&: kernelsTmp); |
| 326 | } |
| 327 | |
| 328 | KernelTableAttr KernelTableAttr::getChecked( |
| 329 | function_ref<InFlightDiagnostic()> emitError, MLIRContext *context, |
| 330 | ArrayRef<KernelMetadataAttr> kernels, bool isSorted) { |
| 331 | // Note that `is_sorted` is always only invoked once even with assertions ON. |
| 332 | assert((!isSorted || llvm::is_sorted(kernels)) && |
| 333 | "expected a sorted kernel array" ); |
| 334 | // Immediately return the attribute if the array is sorted. |
| 335 | if (isSorted || llvm::is_sorted(Range&: kernels)) |
| 336 | return Base::getChecked(emitErrorFn: emitError, ctx&: context, args: kernels); |
| 337 | // Sort the array. |
| 338 | SmallVector<KernelMetadataAttr> kernelsTmp(kernels); |
| 339 | llvm::array_pod_sort(Start: kernelsTmp.begin(), End: kernelsTmp.end()); |
| 340 | return Base::getChecked(emitErrorFn: emitError, ctx&: context, args: kernelsTmp); |
| 341 | } |
| 342 | |
| 343 | LogicalResult |
| 344 | KernelTableAttr::verify(function_ref<InFlightDiagnostic()> emitError, |
| 345 | ArrayRef<KernelMetadataAttr> kernels) { |
| 346 | if (kernels.size() < 2) |
| 347 | return success(); |
| 348 | // Check that the kernels are uniquely named. |
| 349 | if (std::adjacent_find(first: kernels.begin(), last: kernels.end(), |
| 350 | binary_pred: [](KernelMetadataAttr l, KernelMetadataAttr r) { |
| 351 | return l.getName() == r.getName(); |
| 352 | }) != kernels.end()) { |
| 353 | return emitError() << "expected all kernels to be uniquely named" ; |
| 354 | } |
| 355 | return success(); |
| 356 | } |
| 357 | |
| 358 | KernelMetadataAttr KernelTableAttr::lookup(StringRef key) const { |
| 359 | auto [iterator, found] = impl::findAttrSorted(first: begin(), last: end(), name: key); |
| 360 | return found ? *iterator : KernelMetadataAttr(); |
| 361 | } |
| 362 | |
| 363 | KernelMetadataAttr KernelTableAttr::lookup(StringAttr key) const { |
| 364 | auto [iterator, found] = impl::findAttrSorted(first: begin(), last: end(), name: key); |
| 365 | return found ? *iterator : KernelMetadataAttr(); |
| 366 | } |
| 367 | |
| 368 | //===----------------------------------------------------------------------===// |
| 369 | // GPU target options |
| 370 | //===----------------------------------------------------------------------===// |
| 371 | |
| 372 | TargetOptions::TargetOptions( |
| 373 | StringRef toolkitPath, ArrayRef<Attribute> librariesToLink, |
| 374 | StringRef cmdOptions, StringRef elfSection, |
| 375 | CompilationTarget compilationTarget, |
| 376 | function_ref<SymbolTable *()> getSymbolTableCallback, |
| 377 | function_ref<void(llvm::Module &)> initialLlvmIRCallback, |
| 378 | function_ref<void(llvm::Module &)> linkedLlvmIRCallback, |
| 379 | function_ref<void(llvm::Module &)> optimizedLlvmIRCallback, |
| 380 | function_ref<void(StringRef)> isaCallback) |
| 381 | : TargetOptions(TypeID::get<TargetOptions>(), toolkitPath, librariesToLink, |
| 382 | cmdOptions, elfSection, compilationTarget, |
| 383 | getSymbolTableCallback, initialLlvmIRCallback, |
| 384 | linkedLlvmIRCallback, optimizedLlvmIRCallback, |
| 385 | isaCallback) {} |
| 386 | |
| 387 | TargetOptions::TargetOptions( |
| 388 | TypeID typeID, StringRef toolkitPath, ArrayRef<Attribute> librariesToLink, |
| 389 | StringRef cmdOptions, StringRef elfSection, |
| 390 | CompilationTarget compilationTarget, |
| 391 | function_ref<SymbolTable *()> getSymbolTableCallback, |
| 392 | function_ref<void(llvm::Module &)> initialLlvmIRCallback, |
| 393 | function_ref<void(llvm::Module &)> linkedLlvmIRCallback, |
| 394 | function_ref<void(llvm::Module &)> optimizedLlvmIRCallback, |
| 395 | function_ref<void(StringRef)> isaCallback) |
| 396 | : toolkitPath(toolkitPath.str()), librariesToLink(librariesToLink), |
| 397 | cmdOptions(cmdOptions.str()), elfSection(elfSection.str()), |
| 398 | compilationTarget(compilationTarget), |
| 399 | getSymbolTableCallback(getSymbolTableCallback), |
| 400 | initialLlvmIRCallback(initialLlvmIRCallback), |
| 401 | linkedLlvmIRCallback(linkedLlvmIRCallback), |
| 402 | optimizedLlvmIRCallback(optimizedLlvmIRCallback), |
| 403 | isaCallback(isaCallback), typeID(typeID) {} |
| 404 | |
| 405 | TypeID TargetOptions::getTypeID() const { return typeID; } |
| 406 | |
| 407 | StringRef TargetOptions::getToolkitPath() const { return toolkitPath; } |
| 408 | |
| 409 | ArrayRef<Attribute> TargetOptions::getLibrariesToLink() const { |
| 410 | return librariesToLink; |
| 411 | } |
| 412 | |
| 413 | StringRef TargetOptions::getCmdOptions() const { return cmdOptions; } |
| 414 | |
| 415 | StringRef TargetOptions::getELFSection() const { return elfSection; } |
| 416 | |
| 417 | SymbolTable *TargetOptions::getSymbolTable() const { |
| 418 | return getSymbolTableCallback ? getSymbolTableCallback() : nullptr; |
| 419 | } |
| 420 | |
| 421 | function_ref<void(llvm::Module &)> |
| 422 | TargetOptions::getInitialLlvmIRCallback() const { |
| 423 | return initialLlvmIRCallback; |
| 424 | } |
| 425 | |
| 426 | function_ref<void(llvm::Module &)> |
| 427 | TargetOptions::getLinkedLlvmIRCallback() const { |
| 428 | return linkedLlvmIRCallback; |
| 429 | } |
| 430 | |
| 431 | function_ref<void(llvm::Module &)> |
| 432 | TargetOptions::getOptimizedLlvmIRCallback() const { |
| 433 | return optimizedLlvmIRCallback; |
| 434 | } |
| 435 | |
| 436 | function_ref<void(StringRef)> TargetOptions::getISACallback() const { |
| 437 | return isaCallback; |
| 438 | } |
| 439 | |
| 440 | CompilationTarget TargetOptions::getCompilationTarget() const { |
| 441 | return compilationTarget; |
| 442 | } |
| 443 | |
| 444 | CompilationTarget TargetOptions::getDefaultCompilationTarget() { |
| 445 | return CompilationTarget::Fatbin; |
| 446 | } |
| 447 | |
| 448 | std::pair<llvm::BumpPtrAllocator, SmallVector<const char *>> |
| 449 | TargetOptions::tokenizeCmdOptions(const std::string &cmdOptions) { |
| 450 | std::pair<llvm::BumpPtrAllocator, SmallVector<const char *>> options; |
| 451 | llvm::StringSaver stringSaver(options.first); |
| 452 | StringRef opts = cmdOptions; |
| 453 | // For a correct tokenization of the command line options `opts` must be |
| 454 | // unquoted, otherwise the tokenization function returns a single string: the |
| 455 | // unquoted `cmdOptions` -which is not the desired behavior. |
| 456 | // Remove any quotes if they are at the beginning and end of the string: |
| 457 | if (!opts.empty() && opts.front() == '"' && opts.back() == '"') |
| 458 | opts.consume_front(Prefix: "\"" ), opts.consume_back(Suffix: "\"" ); |
| 459 | if (!opts.empty() && opts.front() == '\'' && opts.back() == '\'') |
| 460 | opts.consume_front(Prefix: "'" ), opts.consume_back(Suffix: "'" ); |
| 461 | #ifdef _WIN32 |
| 462 | llvm::cl::TokenizeWindowsCommandLine(opts, stringSaver, options.second, |
| 463 | /*MarkEOLs=*/false); |
| 464 | #else |
| 465 | llvm::cl::TokenizeGNUCommandLine(Source: opts, Saver&: stringSaver, NewArgv&: options.second, |
| 466 | /*MarkEOLs=*/false); |
| 467 | #endif // _WIN32 |
| 468 | return options; |
| 469 | } |
| 470 | |
| 471 | std::pair<llvm::BumpPtrAllocator, SmallVector<const char *>> |
| 472 | TargetOptions::tokenizeCmdOptions() const { |
| 473 | return tokenizeCmdOptions(cmdOptions); |
| 474 | } |
| 475 | |
| 476 | std::pair<llvm::BumpPtrAllocator, SmallVector<const char *>> |
| 477 | TargetOptions::tokenizeAndRemoveSuffixCmdOptions(llvm::StringRef startsWith) { |
| 478 | size_t startPos = cmdOptions.find(svt: startsWith); |
| 479 | if (startPos == std::string::npos) |
| 480 | return {llvm::BumpPtrAllocator(), SmallVector<const char *>()}; |
| 481 | |
| 482 | auto tokenized = |
| 483 | tokenizeCmdOptions(cmdOptions: cmdOptions.substr(pos: startPos + startsWith.size())); |
| 484 | cmdOptions.resize(n: startPos); |
| 485 | return tokenized; |
| 486 | } |
| 487 | |
| 488 | MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::TargetOptions) |
| 489 | |
| 490 | #include "mlir/Dialect/GPU/IR/GPUOpsEnums.cpp.inc" |
| 491 | |
| 492 | #define GET_ATTRDEF_CLASSES |
| 493 | #include "mlir/Dialect/GPU/IR/GPUOpsAttributes.cpp.inc" |
| 494 | |
| 495 | namespace { |
| 496 | template <typename... Attrs> |
| 497 | void registerAttributeStorageImpl(MLIRContext *context) { |
| 498 | (mlir::detail::AttributeUniquer::registerAttribute<Attrs>(context), ...); |
| 499 | } |
| 500 | } // namespace |
| 501 | |
| 502 | void mlir::gpu::detail::registerAttributeStorage(MLIRContext *context) { |
| 503 | registerAttributeStorageImpl< |
| 504 | #define GET_ATTRDEF_LIST |
| 505 | #include "mlir/Dialect/GPU/IR/GPUOpsAttributes.cpp.inc" |
| 506 | >(context); |
| 507 | } |
| 508 | |
| 509 | #include "mlir/Dialect/GPU/IR/CompilationAttrInterfaces.cpp.inc" |
| 510 | |