| 1 | //===- LLVMMemoryOps.cpp - LLVM dialect memory ops ------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file defines the ops of op partition `Memory` of the LLVM dialect. |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #include "mlir/Dialect/LLVMIR/LLVMMemoryOps.h" |
| 14 | |
| 15 | #include "IR/LLVMOps.h" |
| 16 | #include "mlir/Dialect/LLVMIR/LLVMArithOps.h" |
| 17 | #include "mlir/Dialect/LLVMIR/LLVMConstantOps.h" |
| 18 | #include "mlir/Dialect/LLVMIR/LLVMConversionOps.h" |
| 19 | #include "mlir/Dialect/LLVMIR/LLVMDebugIntrinsicOps.h" |
| 20 | #include "mlir/IR/Builders.h" |
| 21 | #include "mlir/IR/DialectImplementation.h" |
| 22 | #include "mlir/IR/Matchers.h" |
| 23 | #include "mlir/IR/PatternMatch.h" |
| 24 | #include "mlir/Interfaces/DataLayoutInterfaces.h" |
| 25 | #include "mlir/Interfaces/MemorySlotInterfaces.h" |
| 26 | #include "llvm/ADT/STLExtras.h" |
| 27 | #include "llvm/ADT/TypeSwitch.h" |
| 28 | #include "llvm/Support/DebugLog.h" |
| 29 | |
| 30 | #include <optional> |
| 31 | |
| 32 | #define DEBUG_TYPE "sroa" |
| 33 | |
| 34 | using namespace mlir; |
| 35 | using namespace mlir::LLVM; |
| 36 | |
| 37 | //===----------------------------------------------------------------------===// |
| 38 | // Attribute Helpers |
| 39 | //===----------------------------------------------------------------------===// |
| 40 | |
| 41 | static constexpr const char kElemTypeAttrName[] = "elem_type" ; |
| 42 | |
| 43 | //===----------------------------------------------------------------------===// |
| 44 | // Printing, parsing, verification and canonicalization for LLVM::AllocaOp. |
| 45 | //===----------------------------------------------------------------------===// |
| 46 | |
| 47 | void AllocaOp::print(OpAsmPrinter &p) { |
| 48 | auto funcTy = |
| 49 | FunctionType::get(context: getContext(), inputs: {getArraySize().getType()}, results: {getType()}); |
| 50 | |
| 51 | if (getInalloca()) |
| 52 | p << " inalloca" ; |
| 53 | |
| 54 | p << ' ' << getArraySize() << " x " << getElemType(); |
| 55 | NamedAttrList attrs((*this)->getDiscardableAttrDictionary().getValue()); |
| 56 | if (getAlignment() && *getAlignment() != 0) |
| 57 | attrs.append(name: getAlignmentAttrName(), attr: getAlignmentAttr()); |
| 58 | p.printOptionalAttrDict(attrs); |
| 59 | p << " : " << funcTy; |
| 60 | } |
| 61 | |
| 62 | // <operation> ::= `llvm.alloca` `inalloca`? ssa-use `x` type |
| 63 | // attribute-dict? `:` type `,` type |
| 64 | ParseResult AllocaOp::parse(OpAsmParser &parser, OperationState &result) { |
| 65 | OpAsmParser::UnresolvedOperand arraySize; |
| 66 | Type type, elemType; |
| 67 | SMLoc trailingTypeLoc; |
| 68 | |
| 69 | if (succeeded(Result: parser.parseOptionalKeyword(keyword: "inalloca" ))) |
| 70 | result.addAttribute(name: getInallocaAttrName(name: result.name), |
| 71 | attr: UnitAttr::get(context: parser.getContext())); |
| 72 | |
| 73 | if (parser.parseOperand(result&: arraySize) || parser.parseKeyword(keyword: "x" ) || |
| 74 | parser.parseType(result&: elemType) || |
| 75 | parser.parseOptionalAttrDict(result&: result.attributes) || parser.parseColon() || |
| 76 | parser.getCurrentLocation(loc: &trailingTypeLoc) || parser.parseType(result&: type)) |
| 77 | return failure(); |
| 78 | |
| 79 | std::optional<NamedAttribute> alignmentAttr = |
| 80 | result.attributes.getNamed(name: "alignment" ); |
| 81 | if (alignmentAttr.has_value()) { |
| 82 | auto alignmentInt = llvm::dyn_cast<IntegerAttr>(Val: alignmentAttr->getValue()); |
| 83 | if (!alignmentInt) |
| 84 | return parser.emitError(loc: parser.getNameLoc(), |
| 85 | message: "expected integer alignment" ); |
| 86 | if (alignmentInt.getValue().isZero()) |
| 87 | result.attributes.erase(name: "alignment" ); |
| 88 | } |
| 89 | |
| 90 | // Extract the result type from the trailing function type. |
| 91 | auto funcType = llvm::dyn_cast<FunctionType>(Val&: type); |
| 92 | if (!funcType || funcType.getNumInputs() != 1 || |
| 93 | funcType.getNumResults() != 1) |
| 94 | return parser.emitError( |
| 95 | loc: trailingTypeLoc, |
| 96 | message: "expected trailing function type with one argument and one result" ); |
| 97 | |
| 98 | if (parser.resolveOperand(operand: arraySize, type: funcType.getInput(i: 0), result&: result.operands)) |
| 99 | return failure(); |
| 100 | |
| 101 | Type resultType = funcType.getResult(i: 0); |
| 102 | if (auto ptrResultType = llvm::dyn_cast<LLVMPointerType>(Val&: resultType)) |
| 103 | result.addAttribute(name: kElemTypeAttrName, attr: TypeAttr::get(type: elemType)); |
| 104 | |
| 105 | result.addTypes(newTypes: {funcType.getResult(i: 0)}); |
| 106 | return success(); |
| 107 | } |
| 108 | |
| 109 | LogicalResult AllocaOp::verify() { |
| 110 | // Only certain target extension types can be used in 'alloca'. |
| 111 | if (auto targetExtType = dyn_cast<LLVMTargetExtType>(Val: getElemType()); |
| 112 | targetExtType && !targetExtType.supportsMemOps()) |
| 113 | return emitOpError() |
| 114 | << "this target extension type cannot be used in alloca" ; |
| 115 | |
| 116 | return success(); |
| 117 | } |
| 118 | |
| 119 | LogicalResult AllocaOp::canonicalize(AllocaOp op, PatternRewriter &rewriter) { |
| 120 | // Convert `alloca Ty, C` to the canonical `alloca [C x Ty], 1` form. |
| 121 | APInt numElements; |
| 122 | if (!matchPattern(value: op.getArraySize(), pattern: m_ConstantInt(bind_value: &numElements)) || |
| 123 | numElements.isOne() || numElements.getActiveBits() > 64) |
| 124 | return failure(); |
| 125 | |
| 126 | auto arrayType = |
| 127 | LLVMArrayType::get(elementType: op.getElemType(), numElements: numElements.getZExtValue()); |
| 128 | Value one = ConstantOp::create(builder&: rewriter, location: op.getLoc(), type: rewriter.getI32Type(), |
| 129 | /*value=*/1); |
| 130 | auto newAlloca = |
| 131 | AllocaOp::create(builder&: rewriter, location: op.getLoc(), res: op.getType(), arraySize: one, |
| 132 | alignment: op.getAlignmentAttr(), elem_type: arrayType, inalloca: op.getInalloca()); |
| 133 | newAlloca->setDiscardableAttrs(op->getDiscardableAttrDictionary()); |
| 134 | rewriter.replaceOp(op, newOp: newAlloca); |
| 135 | return success(); |
| 136 | } |
| 137 | |
| 138 | //===----------------------------------------------------------------------===// |
| 139 | // Code for LLVM::GEPOp. |
| 140 | //===----------------------------------------------------------------------===// |
| 141 | |
| 142 | GEPIndicesAdaptor<ValueRange> GEPOp::getIndices() { |
| 143 | return GEPIndicesAdaptor<ValueRange>(getRawConstantIndicesAttr(), |
| 144 | getDynamicIndices()); |
| 145 | } |
| 146 | |
| 147 | /// Returns the elemental type of any LLVM-compatible vector type or self. |
| 148 | Type mlir::LLVM::(Type type) { |
| 149 | if (auto vectorType = llvm::dyn_cast<VectorType>(Val&: type)) |
| 150 | return vectorType.getElementType(); |
| 151 | return type; |
| 152 | } |
| 153 | |
| 154 | /// Destructures the 'indices' parameter into 'rawConstantIndices' and |
| 155 | /// 'dynamicIndices', encoding the former in the process. In the process, |
| 156 | /// dynamic indices which are used to index into a structure type are converted |
| 157 | /// to constant indices when possible. To do this, the GEPs element type should |
| 158 | /// be passed as first parameter. |
| 159 | static void destructureIndices(Type currType, ArrayRef<GEPArg> indices, |
| 160 | SmallVectorImpl<int32_t> &rawConstantIndices, |
| 161 | SmallVectorImpl<Value> &dynamicIndices) { |
| 162 | for (const GEPArg &iter : indices) { |
| 163 | // If the thing we are currently indexing into is a struct we must turn |
| 164 | // any integer constants into constant indices. If this is not possible |
| 165 | // we don't do anything here. The verifier will catch it and emit a proper |
| 166 | // error. All other canonicalization is done in the fold method. |
| 167 | bool requiresConst = !rawConstantIndices.empty() && |
| 168 | isa_and_nonnull<LLVMStructType>(Val: currType); |
| 169 | if (Value val = llvm::dyn_cast_if_present<Value>(Val: iter)) { |
| 170 | APInt intC; |
| 171 | if (requiresConst && matchPattern(value: val, pattern: m_ConstantInt(bind_value: &intC)) && |
| 172 | intC.isSignedIntN(N: kGEPConstantBitWidth)) { |
| 173 | rawConstantIndices.push_back(Elt: intC.getSExtValue()); |
| 174 | } else { |
| 175 | rawConstantIndices.push_back(Elt: GEPOp::kDynamicIndex); |
| 176 | dynamicIndices.push_back(Elt: val); |
| 177 | } |
| 178 | } else { |
| 179 | rawConstantIndices.push_back(Elt: cast<GEPConstantIndex>(Val: iter)); |
| 180 | } |
| 181 | |
| 182 | // Skip for very first iteration of this loop. First index does not index |
| 183 | // within the aggregates, but is just a pointer offset. |
| 184 | if (rawConstantIndices.size() == 1 || !currType) |
| 185 | continue; |
| 186 | |
| 187 | currType = TypeSwitch<Type, Type>(currType) |
| 188 | .Case<VectorType, LLVMArrayType>(caseFn: [](auto containerType) { |
| 189 | return containerType.getElementType(); |
| 190 | }) |
| 191 | .Case(caseFn: [&](LLVMStructType structType) -> Type { |
| 192 | int64_t memberIndex = rawConstantIndices.back(); |
| 193 | if (memberIndex >= 0 && static_cast<size_t>(memberIndex) < |
| 194 | structType.getBody().size()) |
| 195 | return structType.getBody()[memberIndex]; |
| 196 | return nullptr; |
| 197 | }) |
| 198 | .Default(nullptr); |
| 199 | } |
| 200 | } |
| 201 | |
| 202 | void GEPOp::build(OpBuilder &builder, OperationState &result, Type resultType, |
| 203 | Type elementType, Value basePtr, ArrayRef<GEPArg> indices, |
| 204 | GEPNoWrapFlags noWrapFlags, ConstantRangeAttr inrange, |
| 205 | ArrayRef<NamedAttribute> attributes) { |
| 206 | SmallVector<int32_t> rawConstantIndices; |
| 207 | SmallVector<Value> dynamicIndices; |
| 208 | destructureIndices(currType: elementType, indices, rawConstantIndices, dynamicIndices); |
| 209 | |
| 210 | result.addTypes(newTypes: resultType); |
| 211 | result.addAttributes(newAttributes: attributes); |
| 212 | result.getOrAddProperties<Properties>().rawConstantIndices = |
| 213 | builder.getDenseI32ArrayAttr(values: rawConstantIndices); |
| 214 | result.getOrAddProperties<Properties>().noWrapFlags = noWrapFlags; |
| 215 | result.getOrAddProperties<Properties>().elem_type = |
| 216 | TypeAttr::get(type: elementType); |
| 217 | result.getOrAddProperties<Properties>().inrange = inrange; |
| 218 | result.addOperands(newOperands: basePtr); |
| 219 | result.addOperands(newOperands: dynamicIndices); |
| 220 | } |
| 221 | |
| 222 | void GEPOp::build(OpBuilder &builder, OperationState &result, Type resultType, |
| 223 | Type elementType, Value basePtr, ValueRange indices, |
| 224 | GEPNoWrapFlags noWrapFlags, ConstantRangeAttr inrange, |
| 225 | ArrayRef<NamedAttribute> attributes) { |
| 226 | build(builder, result, resultType, elementType, basePtr, |
| 227 | indices: SmallVector<GEPArg>(indices), noWrapFlags, inrange, attributes); |
| 228 | } |
| 229 | |
| 230 | ParseResult mlir::LLVM::parseGEPIndices( |
| 231 | OpAsmParser &parser, |
| 232 | SmallVectorImpl<OpAsmParser::UnresolvedOperand> &indices, |
| 233 | DenseI32ArrayAttr &rawConstantIndices) { |
| 234 | SmallVector<int32_t> constantIndices; |
| 235 | |
| 236 | auto idxParser = [&]() -> ParseResult { |
| 237 | int32_t constantIndex; |
| 238 | OptionalParseResult parsedInteger = |
| 239 | parser.parseOptionalInteger(result&: constantIndex); |
| 240 | if (parsedInteger.has_value()) { |
| 241 | if (failed(Result: parsedInteger.value())) |
| 242 | return failure(); |
| 243 | constantIndices.push_back(Elt: constantIndex); |
| 244 | return success(); |
| 245 | } |
| 246 | |
| 247 | constantIndices.push_back(Elt: LLVM::GEPOp::kDynamicIndex); |
| 248 | return parser.parseOperand(result&: indices.emplace_back()); |
| 249 | }; |
| 250 | if (parser.parseCommaSeparatedList(parseElementFn: idxParser)) |
| 251 | return failure(); |
| 252 | |
| 253 | rawConstantIndices = |
| 254 | DenseI32ArrayAttr::get(context: parser.getContext(), content: constantIndices); |
| 255 | return success(); |
| 256 | } |
| 257 | |
| 258 | void mlir::LLVM::printGEPIndices(OpAsmPrinter &printer, LLVM::GEPOp gepOp, |
| 259 | OperandRange indices, |
| 260 | DenseI32ArrayAttr rawConstantIndices) { |
| 261 | llvm::interleaveComma( |
| 262 | c: GEPIndicesAdaptor<OperandRange>(rawConstantIndices, indices), os&: printer, |
| 263 | each_fn: [&](PointerUnion<IntegerAttr, Value> cst) { |
| 264 | if (Value val = llvm::dyn_cast_if_present<Value>(Val&: cst)) |
| 265 | printer.printOperand(value: val); |
| 266 | else |
| 267 | printer << cast<IntegerAttr>(Val&: cst).getInt(); |
| 268 | }); |
| 269 | } |
| 270 | |
| 271 | /// For the given `indices`, check if they comply with `baseGEPType`, |
| 272 | /// especially check against LLVMStructTypes nested within. |
| 273 | static LogicalResult |
| 274 | verifyStructIndices(Type baseGEPType, unsigned indexPos, |
| 275 | GEPIndicesAdaptor<ValueRange> indices, |
| 276 | function_ref<InFlightDiagnostic()> emitOpError) { |
| 277 | if (indexPos >= indices.size()) |
| 278 | // Stop searching |
| 279 | return success(); |
| 280 | |
| 281 | return TypeSwitch<Type, LogicalResult>(baseGEPType) |
| 282 | .Case(caseFn: [&](LLVMStructType structType) -> LogicalResult { |
| 283 | auto attr = dyn_cast<IntegerAttr>(Val: indices[indexPos]); |
| 284 | if (!attr) |
| 285 | return emitOpError() << "expected index " << indexPos |
| 286 | << " indexing a struct to be constant" ; |
| 287 | |
| 288 | int32_t gepIndex = attr.getInt(); |
| 289 | ArrayRef<Type> elementTypes = structType.getBody(); |
| 290 | if (gepIndex < 0 || |
| 291 | static_cast<size_t>(gepIndex) >= elementTypes.size()) |
| 292 | return emitOpError() << "index " << indexPos |
| 293 | << " indexing a struct is out of bounds" ; |
| 294 | |
| 295 | // Instead of recursively going into every children types, we only |
| 296 | // dive into the one indexed by gepIndex. |
| 297 | return verifyStructIndices(baseGEPType: elementTypes[gepIndex], indexPos: indexPos + 1, |
| 298 | indices, emitOpError); |
| 299 | }) |
| 300 | .Case<VectorType, LLVMArrayType>( |
| 301 | caseFn: [&](auto containerType) -> LogicalResult { |
| 302 | return verifyStructIndices(containerType.getElementType(), |
| 303 | indexPos + 1, indices, emitOpError); |
| 304 | }) |
| 305 | .Default(defaultFn: [&](auto otherType) -> LogicalResult { |
| 306 | return emitOpError() |
| 307 | << "type " << otherType << " cannot be indexed (index #" |
| 308 | << indexPos << ")" ; |
| 309 | }); |
| 310 | } |
| 311 | |
| 312 | /// Driver function around `verifyStructIndices`. |
| 313 | static LogicalResult |
| 314 | verifyStructIndices(Type baseGEPType, GEPIndicesAdaptor<ValueRange> indices, |
| 315 | function_ref<InFlightDiagnostic()> emitOpError) { |
| 316 | return verifyStructIndices(baseGEPType, /*indexPos=*/1, indices, emitOpError); |
| 317 | } |
| 318 | |
| 319 | LogicalResult LLVM::GEPOp::verify() { |
| 320 | if (static_cast<size_t>( |
| 321 | llvm::count(Range: getRawConstantIndices(), Element: kDynamicIndex)) != |
| 322 | getDynamicIndices().size()) |
| 323 | return emitOpError(message: "expected as many dynamic indices as specified in '" ) |
| 324 | << getRawConstantIndicesAttrName().getValue() << "'" ; |
| 325 | |
| 326 | if (getNoWrapFlags() == GEPNoWrapFlags::inboundsFlag) |
| 327 | return emitOpError(message: "'inbounds_flag' cannot be used directly." ); |
| 328 | |
| 329 | // LLVM stores `inrange` only on GetElementPtrConstantExpr, whose operands |
| 330 | // are already Constants. This dialect has one GEP operation, so proving the |
| 331 | // base and indices are constant expressions means walking SSA. A local check |
| 332 | // rejects valid cases such as a nested constant GEP base or a `ptrtoint` |
| 333 | // index. |
| 334 | if (auto inrange = getInrangeAttr()) { |
| 335 | auto pointerType = |
| 336 | cast<LLVMPointerType>(Val: extractVectorElementType(type: getBase().getType())); |
| 337 | DataLayout dataLayout = DataLayout::closest(op: *this); |
| 338 | std::optional<uint64_t> indexWidth = |
| 339 | dataLayout.getTypeIndexBitwidth(t: pointerType); |
| 340 | assert(indexWidth && "pointers always return an index bitwidth" ); |
| 341 | if (inrange.getLower().getBitWidth() != *indexWidth) |
| 342 | return emitOpError(message: "'inrange' bitwidth " ) |
| 343 | << inrange.getLower().getBitWidth() |
| 344 | << " must match the pointer index bitwidth (" << *indexWidth |
| 345 | << ") specified in the datalayout" ; |
| 346 | if (inrange.getLower().sge(RHS: inrange.getUpper())) |
| 347 | return emitOpError(message: "expected 'inrange' end to be larger than start" ); |
| 348 | } |
| 349 | |
| 350 | return verifyStructIndices(baseGEPType: getElemType(), indices: getIndices(), |
| 351 | emitOpError: [&] { return emitOpError(); }); |
| 352 | } |
| 353 | |
| 354 | //===----------------------------------------------------------------------===// |
| 355 | // LoadOp |
| 356 | //===----------------------------------------------------------------------===// |
| 357 | |
| 358 | void LoadOp::getEffects( |
| 359 | SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>> |
| 360 | &effects) { |
| 361 | effects.emplace_back(Args: MemoryEffects::Read::get(), Args: &getAddrMutable()); |
| 362 | // Volatile operations can have target-specific read-write effects on |
| 363 | // memory besides the one referred to by the pointer operand. |
| 364 | // Similarly, atomic operations that are monotonic or stricter cause |
| 365 | // synchronization that from a language point-of-view, are arbitrary |
| 366 | // read-writes into memory. |
| 367 | if (getVolatile_() || (getOrdering() != AtomicOrdering::not_atomic && |
| 368 | getOrdering() != AtomicOrdering::unordered)) { |
| 369 | effects.emplace_back(Args: MemoryEffects::Write::get()); |
| 370 | effects.emplace_back(Args: MemoryEffects::Read::get()); |
| 371 | } |
| 372 | } |
| 373 | |
| 374 | /// Returns true if the given type is supported by atomic operations. All |
| 375 | /// integer, float, and pointer types with a power-of-two bitsize and a minimal |
| 376 | /// size of 8 bits are supported. |
| 377 | bool LLVM::isTypeCompatibleWithAtomicOp(Type type, |
| 378 | const DataLayout &dataLayout) { |
| 379 | if (!isa<IntegerType, LLVMPointerType>(Val: type)) |
| 380 | if (!isCompatibleFloatingPointType(type)) |
| 381 | return false; |
| 382 | |
| 383 | llvm::TypeSize bitWidth = dataLayout.getTypeSizeInBits(t: type); |
| 384 | if (bitWidth.isScalable()) |
| 385 | return false; |
| 386 | // Needs to be at least 8 bits and a power of two. |
| 387 | return bitWidth >= 8 && (bitWidth & (bitWidth - 1)) == 0; |
| 388 | } |
| 389 | |
| 390 | /// Verifies the attributes and the type of atomic memory access operations. |
| 391 | template <typename OpTy> |
| 392 | static LogicalResult |
| 393 | verifyAtomicMemOp(OpTy memOp, Type valueType, |
| 394 | ArrayRef<AtomicOrdering> unsupportedOrderings) { |
| 395 | if (memOp.getOrdering() != AtomicOrdering::not_atomic) { |
| 396 | DataLayout dataLayout = DataLayout::closest(op: memOp); |
| 397 | if (!isTypeCompatibleWithAtomicOp(type: valueType, dataLayout)) |
| 398 | return memOp.emitOpError("unsupported type " ) |
| 399 | << valueType << " for atomic access" ; |
| 400 | if (llvm::is_contained(unsupportedOrderings, memOp.getOrdering())) |
| 401 | return memOp.emitOpError("unsupported ordering '" ) |
| 402 | << stringifyAtomicOrdering(memOp.getOrdering()) << "'" ; |
| 403 | if (!memOp.getAlignment()) |
| 404 | return memOp.emitOpError("expected alignment for atomic access" ); |
| 405 | return success(); |
| 406 | } |
| 407 | if (memOp.getSyncscope()) |
| 408 | return memOp.emitOpError( |
| 409 | "expected syncscope to be null for non-atomic access" ); |
| 410 | return success(); |
| 411 | } |
| 412 | |
| 413 | LogicalResult LoadOp::verify() { |
| 414 | Type valueType = getResult().getType(); |
| 415 | return verifyAtomicMemOp(memOp: *this, valueType, |
| 416 | unsupportedOrderings: {AtomicOrdering::release, AtomicOrdering::acq_rel}); |
| 417 | } |
| 418 | |
| 419 | void LoadOp::build(OpBuilder &builder, OperationState &state, Type type, |
| 420 | Value addr, unsigned alignment, bool isVolatile, |
| 421 | bool isNonTemporal, bool isInvariant, bool isInvariantGroup, |
| 422 | bool isNoundef, AtomicOrdering ordering, |
| 423 | StringRef syncscope) { |
| 424 | build(odsBuilder&: builder, odsState&: state, res: type, addr, |
| 425 | alignment: alignment ? builder.getI64IntegerAttr(value: alignment) : nullptr, volatile_: isVolatile, |
| 426 | nontemporal: isNonTemporal, invariant: isInvariant, invariantGroup: isInvariantGroup, ordering, |
| 427 | syncscope: syncscope.empty() ? nullptr : builder.getStringAttr(bytes: syncscope), |
| 428 | /*dereferenceable=*/nullptr, noundef: isNoundef, /*range=*/nullptr, |
| 429 | /*access_groups=*/nullptr, |
| 430 | /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, |
| 431 | /*tbaa=*/nullptr); |
| 432 | } |
| 433 | |
| 434 | //===----------------------------------------------------------------------===// |
| 435 | // StoreOp |
| 436 | //===----------------------------------------------------------------------===// |
| 437 | |
| 438 | void StoreOp::getEffects( |
| 439 | SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>> |
| 440 | &effects) { |
| 441 | effects.emplace_back(Args: MemoryEffects::Write::get(), Args: &getAddrMutable()); |
| 442 | // Volatile operations can have target-specific read-write effects on |
| 443 | // memory besides the one referred to by the pointer operand. |
| 444 | // Similarly, atomic operations that are monotonic or stricter cause |
| 445 | // synchronization that from a language point-of-view, are arbitrary |
| 446 | // read-writes into memory. |
| 447 | if (getVolatile_() || (getOrdering() != AtomicOrdering::not_atomic && |
| 448 | getOrdering() != AtomicOrdering::unordered)) { |
| 449 | effects.emplace_back(Args: MemoryEffects::Write::get()); |
| 450 | effects.emplace_back(Args: MemoryEffects::Read::get()); |
| 451 | } |
| 452 | } |
| 453 | |
| 454 | LogicalResult StoreOp::verify() { |
| 455 | Type valueType = getValue().getType(); |
| 456 | return verifyAtomicMemOp(memOp: *this, valueType, |
| 457 | unsupportedOrderings: {AtomicOrdering::acquire, AtomicOrdering::acq_rel}); |
| 458 | } |
| 459 | |
| 460 | void StoreOp::build(OpBuilder &builder, OperationState &state, Value value, |
| 461 | Value addr, unsigned alignment, bool isVolatile, |
| 462 | bool isNonTemporal, bool isInvariantGroup, |
| 463 | AtomicOrdering ordering, StringRef syncscope) { |
| 464 | build(odsBuilder&: builder, odsState&: state, value, addr, |
| 465 | alignment: alignment ? builder.getI64IntegerAttr(value: alignment) : nullptr, volatile_: isVolatile, |
| 466 | nontemporal: isNonTemporal, invariantGroup: isInvariantGroup, ordering, |
| 467 | syncscope: syncscope.empty() ? nullptr : builder.getStringAttr(bytes: syncscope), |
| 468 | /*access_groups=*/nullptr, |
| 469 | /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr); |
| 470 | } |
| 471 | |
| 472 | //===----------------------------------------------------------------------===// |
| 473 | // AtomicRMWOp |
| 474 | //===----------------------------------------------------------------------===// |
| 475 | |
| 476 | void AtomicRMWOp::build(OpBuilder &builder, OperationState &state, |
| 477 | AtomicBinOp binOp, Value ptr, Value val, |
| 478 | AtomicOrdering ordering, StringRef syncscope, |
| 479 | unsigned alignment, bool isVolatile, |
| 480 | bool ignoreDenormalMode) { |
| 481 | build(odsBuilder&: builder, odsState&: state, res: val.getType(), bin_op: binOp, ptr, val, ordering, |
| 482 | syncscope: !syncscope.empty() ? builder.getStringAttr(bytes: syncscope) : nullptr, |
| 483 | alignment: alignment ? builder.getI64IntegerAttr(value: alignment) : nullptr, volatile_: isVolatile, |
| 484 | ignore_denormal_mode: ignoreDenormalMode, |
| 485 | /*access_groups=*/nullptr, |
| 486 | /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr); |
| 487 | } |
| 488 | |
| 489 | LogicalResult AtomicRMWOp::verify() { |
| 490 | auto valType = getVal().getType(); |
| 491 | if (isFloatingPointOperation()) { |
| 492 | if (isCompatibleVectorType(type: valType)) { |
| 493 | if (isScalableVectorType(vectorType: valType)) |
| 494 | return emitOpError(message: "expected LLVM IR fixed vector type" ); |
| 495 | Type elemType = llvm::cast<VectorType>(Val&: valType).getElementType(); |
| 496 | if (!isCompatibleFloatingPointType(type: elemType)) |
| 497 | return emitOpError( |
| 498 | message: "expected LLVM IR floating point type for vector element" ); |
| 499 | } else if (!isCompatibleFloatingPointType(type: valType)) { |
| 500 | return emitOpError(message: "expected LLVM IR floating point type" ); |
| 501 | } |
| 502 | } else if (getBinOp() == AtomicBinOp::xchg) { |
| 503 | DataLayout dataLayout = DataLayout::closest(op: *this); |
| 504 | if (!isTypeCompatibleWithAtomicOp(type: valType, dataLayout)) |
| 505 | return emitOpError(message: "unexpected LLVM IR type for 'xchg' bin_op" ); |
| 506 | } else { |
| 507 | auto intType = llvm::dyn_cast<IntegerType>(Val&: valType); |
| 508 | unsigned intBitWidth = intType ? intType.getWidth() : 0; |
| 509 | if (intBitWidth != 8 && intBitWidth != 16 && intBitWidth != 32 && |
| 510 | intBitWidth != 64) |
| 511 | return emitOpError(message: "expected LLVM IR integer type" ); |
| 512 | } |
| 513 | |
| 514 | if (getIgnoreDenormalMode() && !isFloatingPointOperation()) |
| 515 | return emitOpError( |
| 516 | message: "expected floating-point operation with 'ignore_denormal_mode'" ); |
| 517 | |
| 518 | if (static_cast<unsigned>(getOrdering()) < |
| 519 | static_cast<unsigned>(AtomicOrdering::monotonic)) |
| 520 | return emitOpError() << "expected at least '" |
| 521 | << stringifyAtomicOrdering(AtomicOrdering::monotonic) |
| 522 | << "' ordering" ; |
| 523 | |
| 524 | return success(); |
| 525 | } |
| 526 | |
| 527 | //===----------------------------------------------------------------------===// |
| 528 | // AtomicCmpXchgOp |
| 529 | //===----------------------------------------------------------------------===// |
| 530 | |
| 531 | /// Returns an LLVM struct type that contains a value type and a boolean type. |
| 532 | LLVMStructType mlir::LLVM::getValAndBoolStructType(Type valType) { |
| 533 | auto boolType = IntegerType::get(context: valType.getContext(), width: 1); |
| 534 | return LLVMStructType::getLiteral(context: valType.getContext(), types: {valType, boolType}); |
| 535 | } |
| 536 | |
| 537 | void AtomicCmpXchgOp::build(OpBuilder &builder, OperationState &state, |
| 538 | Value ptr, Value cmp, Value val, |
| 539 | AtomicOrdering successOrdering, |
| 540 | AtomicOrdering failureOrdering, StringRef syncscope, |
| 541 | unsigned alignment, bool isWeak, bool isVolatile) { |
| 542 | build(odsBuilder&: builder, odsState&: state, res: getValAndBoolStructType(valType: val.getType()), ptr, cmp, val, |
| 543 | success_ordering: successOrdering, failure_ordering: failureOrdering, |
| 544 | syncscope: !syncscope.empty() ? builder.getStringAttr(bytes: syncscope) : nullptr, |
| 545 | alignment: alignment ? builder.getI64IntegerAttr(value: alignment) : nullptr, weak: isWeak, |
| 546 | volatile_: isVolatile, /*access_groups=*/nullptr, |
| 547 | /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr); |
| 548 | } |
| 549 | |
| 550 | LogicalResult AtomicCmpXchgOp::verify() { |
| 551 | auto ptrType = llvm::cast<LLVM::LLVMPointerType>(Val: getPtr().getType()); |
| 552 | if (!ptrType) |
| 553 | return emitOpError(message: "expected LLVM IR pointer type for operand #0" ); |
| 554 | auto valType = getVal().getType(); |
| 555 | DataLayout dataLayout = DataLayout::closest(op: *this); |
| 556 | if (!isTypeCompatibleWithAtomicOp(type: valType, dataLayout)) |
| 557 | return emitOpError(message: "unexpected LLVM IR type" ); |
| 558 | if (getSuccessOrdering() < AtomicOrdering::monotonic || |
| 559 | getFailureOrdering() < AtomicOrdering::monotonic) |
| 560 | return emitOpError(message: "ordering must be at least 'monotonic'" ); |
| 561 | if (getFailureOrdering() == AtomicOrdering::release || |
| 562 | getFailureOrdering() == AtomicOrdering::acq_rel) |
| 563 | return emitOpError(message: "failure ordering cannot be 'release' or 'acq_rel'" ); |
| 564 | return success(); |
| 565 | } |
| 566 | |
| 567 | //===----------------------------------------------------------------------===// |
| 568 | // FenceOp |
| 569 | //===----------------------------------------------------------------------===// |
| 570 | |
| 571 | void FenceOp::build(OpBuilder &builder, OperationState &state, |
| 572 | AtomicOrdering ordering, StringRef syncscope) { |
| 573 | build(odsBuilder&: builder, odsState&: state, ordering, |
| 574 | syncscope: syncscope.empty() ? nullptr : builder.getStringAttr(bytes: syncscope)); |
| 575 | } |
| 576 | |
| 577 | LogicalResult FenceOp::verify() { |
| 578 | if (getOrdering() == AtomicOrdering::not_atomic || |
| 579 | getOrdering() == AtomicOrdering::unordered || |
| 580 | getOrdering() == AtomicOrdering::monotonic) |
| 581 | return emitOpError(message: "can be given only acquire, release, acq_rel, " |
| 582 | "and seq_cst orderings" ); |
| 583 | return success(); |
| 584 | } |
| 585 | |
| 586 | //===----------------------------------------------------------------------===// |
| 587 | // Folder for LLVM::GEPOp |
| 588 | //===----------------------------------------------------------------------===// |
| 589 | |
| 590 | OpFoldResult LLVM::GEPOp::fold(FoldAdaptor adaptor) { |
| 591 | GEPIndicesAdaptor<ArrayRef<Attribute>> indices(getRawConstantIndicesAttr(), |
| 592 | adaptor.getDynamicIndices()); |
| 593 | |
| 594 | // Avoid losing inrange information. |
| 595 | if (getInrangeAttr()) |
| 596 | return {}; |
| 597 | |
| 598 | // gep %x:T, 0 -> %x |
| 599 | if (getBase().getType() == getType() && indices.size() == 1) |
| 600 | if (auto integer = llvm::dyn_cast_or_null<IntegerAttr>(Val: indices[0])) |
| 601 | if (integer.getValue().isZero()) |
| 602 | return getBase(); |
| 603 | |
| 604 | // Canonicalize any dynamic indices of constant value to constant indices. |
| 605 | bool changed = false; |
| 606 | SmallVector<GEPArg> gepArgs; |
| 607 | for (auto iter : llvm::enumerate(First&: indices)) { |
| 608 | auto integer = llvm::dyn_cast_or_null<IntegerAttr>(Val&: iter.value()); |
| 609 | // Constant indices can only be int32_t, so if integer does not fit we |
| 610 | // are forced to keep it dynamic, despite being a constant. |
| 611 | if (!indices.isDynamicIndex(index: iter.index()) || !integer || |
| 612 | !integer.getValue().isSignedIntN(N: kGEPConstantBitWidth)) { |
| 613 | |
| 614 | PointerUnion<IntegerAttr, Value> existing = getIndices()[iter.index()]; |
| 615 | if (Value val = llvm::dyn_cast_if_present<Value>(Val&: existing)) |
| 616 | gepArgs.emplace_back(Args&: val); |
| 617 | else |
| 618 | gepArgs.emplace_back(Args: cast<IntegerAttr>(Val&: existing).getInt()); |
| 619 | |
| 620 | continue; |
| 621 | } |
| 622 | |
| 623 | changed = true; |
| 624 | gepArgs.emplace_back(Args: integer.getInt()); |
| 625 | } |
| 626 | if (changed) { |
| 627 | SmallVector<int32_t> rawConstantIndices; |
| 628 | SmallVector<Value> dynamicIndices; |
| 629 | destructureIndices(currType: getElemType(), indices: gepArgs, rawConstantIndices, |
| 630 | dynamicIndices); |
| 631 | |
| 632 | getDynamicIndicesMutable().assign(values: dynamicIndices); |
| 633 | setRawConstantIndices(rawConstantIndices); |
| 634 | return Value{*this}; |
| 635 | } |
| 636 | |
| 637 | return {}; |
| 638 | } |
| 639 | |
| 640 | Value LLVM::GEPOp::getViewSource() { return getBase(); } |
| 641 | |
| 642 | //===----------------------------------------------------------------------===// |
| 643 | // Interfaces for AllocaOp |
| 644 | //===----------------------------------------------------------------------===// |
| 645 | |
| 646 | llvm::SmallVector<MemorySlot> LLVM::AllocaOp::getPromotableSlots() { |
| 647 | if (!getOperation()->getBlock()->isEntryBlock()) |
| 648 | return {}; |
| 649 | |
| 650 | return {MemorySlot{.ptr: getResult(), .valueType: getElemType()}}; |
| 651 | } |
| 652 | |
| 653 | Value LLVM::AllocaOp::getDefaultValue(const MemorySlot &slot, |
| 654 | OpBuilder &builder) { |
| 655 | return LLVM::UndefOp::create(builder, location: getLoc(), res: slot.valueType); |
| 656 | } |
| 657 | |
| 658 | void LLVM::AllocaOp::handleBlockArgument(const MemorySlot &slot, |
| 659 | BlockArgument argument, |
| 660 | OpBuilder &builder) { |
| 661 | for (Operation *user : getOperation()->getUsers()) |
| 662 | if (auto declareOp = llvm::dyn_cast<LLVM::DbgDeclareOp>(Val: user)) |
| 663 | LLVM::DbgValueOp::create(builder, location: declareOp.getLoc(), value: argument, |
| 664 | varInfo: declareOp.getVarInfo(), |
| 665 | locationExpr: declareOp.getLocationExpr()); |
| 666 | } |
| 667 | |
| 668 | std::optional<PromotableAllocationOpInterface> |
| 669 | LLVM::AllocaOp::handlePromotionComplete(const MemorySlot &slot, |
| 670 | Value defaultValue, |
| 671 | OpBuilder &builder) { |
| 672 | if (defaultValue && defaultValue.use_empty()) |
| 673 | defaultValue.getDefiningOp()->erase(); |
| 674 | this->erase(); |
| 675 | return std::nullopt; |
| 676 | } |
| 677 | |
| 678 | SmallVector<DestructurableMemorySlot> LLVM::AllocaOp::getDestructurableSlots() { |
| 679 | if (!mlir::matchPattern(value: getArraySize(), pattern: m_One())) |
| 680 | return {}; |
| 681 | |
| 682 | auto destructurable = dyn_cast<DestructurableTypeInterface>(Val: getElemType()); |
| 683 | if (!destructurable) |
| 684 | return {}; |
| 685 | |
| 686 | std::optional<DenseMap<Attribute, Type>> destructuredType = |
| 687 | destructurable.getSubelementIndexMap(); |
| 688 | if (!destructuredType) |
| 689 | return {}; |
| 690 | |
| 691 | return {DestructurableMemorySlot{{.ptr: getResult(), .valueType: getElemType()}, |
| 692 | .subelementTypes: *destructuredType}}; |
| 693 | } |
| 694 | |
| 695 | DenseMap<Attribute, MemorySlot> LLVM::AllocaOp::destructure( |
| 696 | const DestructurableMemorySlot &slot, |
| 697 | const SmallPtrSetImpl<Attribute> &usedIndices, OpBuilder &builder, |
| 698 | SmallVectorImpl<DestructurableAllocationOpInterface> &newAllocators) { |
| 699 | assert(slot.ptr == getResult()); |
| 700 | builder.setInsertionPointAfter(*this); |
| 701 | |
| 702 | auto destructurableType = cast<DestructurableTypeInterface>(Val: getElemType()); |
| 703 | DenseMap<Attribute, MemorySlot> slotMap; |
| 704 | // Iterate subelements in their original type order to produce allocas in a |
| 705 | // deterministic, readable order (matching appearance in the source type). |
| 706 | Type i32 = IntegerType::get(context: getContext(), width: 32); |
| 707 | for (size_t i = 0; i < slot.subelementTypes.size(); i++) { |
| 708 | Attribute index = IntegerAttr::get(type: i32, value: i); |
| 709 | if (!usedIndices.contains(Ptr: index)) |
| 710 | continue; |
| 711 | Type elemType = destructurableType.getTypeAtIndex(index); |
| 712 | assert(elemType && "used index must exist" ); |
| 713 | auto subAlloca = LLVM::AllocaOp::create( |
| 714 | builder, location: getLoc(), resultType: LLVM::LLVMPointerType::get(context: getContext()), elementType: elemType, |
| 715 | arraySize: getArraySize()); |
| 716 | newAllocators.push_back(Elt: subAlloca); |
| 717 | slotMap.try_emplace<MemorySlot>(Key: index, Args: {.ptr: subAlloca.getResult(), .valueType: elemType}); |
| 718 | } |
| 719 | |
| 720 | return slotMap; |
| 721 | } |
| 722 | |
| 723 | std::optional<DestructurableAllocationOpInterface> |
| 724 | LLVM::AllocaOp::handleDestructuringComplete( |
| 725 | const DestructurableMemorySlot &slot, OpBuilder &builder) { |
| 726 | assert(slot.ptr == getResult()); |
| 727 | this->erase(); |
| 728 | return std::nullopt; |
| 729 | } |
| 730 | |
| 731 | //===----------------------------------------------------------------------===// |
| 732 | // Interfaces for LoadOp/StoreOp |
| 733 | //===----------------------------------------------------------------------===// |
| 734 | |
| 735 | bool LLVM::LoadOp::loadsFrom(const MemorySlot &slot) { |
| 736 | return getAddr() == slot.ptr; |
| 737 | } |
| 738 | |
| 739 | bool LLVM::LoadOp::storesTo(const MemorySlot &slot) { return false; } |
| 740 | |
| 741 | Value LLVM::LoadOp::getStored(const MemorySlot &slot, OpBuilder &builder, |
| 742 | Value reachingDef, const DataLayout &dataLayout) { |
| 743 | llvm_unreachable("getStored should not be called on LoadOp" ); |
| 744 | } |
| 745 | |
| 746 | bool LLVM::StoreOp::loadsFrom(const MemorySlot &slot) { return false; } |
| 747 | |
| 748 | bool LLVM::StoreOp::storesTo(const MemorySlot &slot) { |
| 749 | return getAddr() == slot.ptr; |
| 750 | } |
| 751 | |
| 752 | /// Checks if `type` can be used in any kind of conversion sequences. |
| 753 | static bool isSupportedTypeForConversion(Type type) { |
| 754 | // Aggregate types are not bitcastable. |
| 755 | if (isa<LLVM::LLVMStructType, LLVM::LLVMArrayType>(Val: type)) |
| 756 | return false; |
| 757 | |
| 758 | if (auto vectorType = dyn_cast<VectorType>(Val&: type)) { |
| 759 | // Vectors of pointers cannot be casted. |
| 760 | if (isa<LLVM::LLVMPointerType>(Val: vectorType.getElementType())) |
| 761 | return false; |
| 762 | // Scalable types are not supported. |
| 763 | return !vectorType.isScalable(); |
| 764 | } |
| 765 | return true; |
| 766 | } |
| 767 | |
| 768 | /// Checks that `rhs` can be converted to `lhs` by a sequence of casts and |
| 769 | /// truncations. Checks for narrowing or widening conversion compatibility |
| 770 | /// depending on `narrowingConversion`. |
| 771 | static bool areConversionCompatible(const DataLayout &layout, Type targetType, |
| 772 | Type srcType, bool narrowingConversion) { |
| 773 | if (targetType == srcType) |
| 774 | return true; |
| 775 | |
| 776 | if (!isSupportedTypeForConversion(type: targetType) || |
| 777 | !isSupportedTypeForConversion(type: srcType)) |
| 778 | return false; |
| 779 | |
| 780 | uint64_t targetSize = layout.getTypeSize(t: targetType); |
| 781 | uint64_t srcSize = layout.getTypeSize(t: srcType); |
| 782 | |
| 783 | // Pointer casts will only be sane when the bitsize of both pointer types is |
| 784 | // the same. |
| 785 | if (isa<LLVM::LLVMPointerType>(Val: targetType) && |
| 786 | isa<LLVM::LLVMPointerType>(Val: srcType)) |
| 787 | return targetSize == srcSize; |
| 788 | |
| 789 | if (narrowingConversion) |
| 790 | return targetSize <= srcSize; |
| 791 | return targetSize >= srcSize; |
| 792 | } |
| 793 | |
| 794 | /// Checks if `dataLayout` describes a little endian layout. |
| 795 | static bool isBigEndian(const DataLayout &dataLayout) { |
| 796 | auto endiannessStr = dyn_cast_or_null<StringAttr>(Val: dataLayout.getEndianness()); |
| 797 | return endiannessStr && endiannessStr == "big" ; |
| 798 | } |
| 799 | |
| 800 | /// Converts a value to an integer type of the same size. |
| 801 | /// Assumes that the type can be converted. |
| 802 | static Value castToSameSizedInt(OpBuilder &builder, Location loc, Value val, |
| 803 | const DataLayout &dataLayout) { |
| 804 | Type type = val.getType(); |
| 805 | assert(isSupportedTypeForConversion(type) && |
| 806 | "expected value to have a convertible type" ); |
| 807 | |
| 808 | if (isa<IntegerType>(Val: type)) |
| 809 | return val; |
| 810 | |
| 811 | uint64_t typeBitSize = dataLayout.getTypeSizeInBits(t: type); |
| 812 | IntegerType valueSizeInteger = builder.getIntegerType(width: typeBitSize); |
| 813 | |
| 814 | if (isa<LLVM::LLVMPointerType>(Val: type)) |
| 815 | return builder.createOrFold<LLVM::PtrToIntOp>(location: loc, args&: valueSizeInteger, args&: val); |
| 816 | return builder.createOrFold<LLVM::BitcastOp>(location: loc, args&: valueSizeInteger, args&: val); |
| 817 | } |
| 818 | |
| 819 | /// Converts a value with an integer type to `targetType`. |
| 820 | static Value castIntValueToSameSizedType(OpBuilder &builder, Location loc, |
| 821 | Value val, Type targetType) { |
| 822 | assert(isa<IntegerType>(val.getType()) && |
| 823 | "expected value to have an integer type" ); |
| 824 | assert(isSupportedTypeForConversion(targetType) && |
| 825 | "expected the target type to be supported for conversions" ); |
| 826 | if (val.getType() == targetType) |
| 827 | return val; |
| 828 | if (isa<LLVM::LLVMPointerType>(Val: targetType)) |
| 829 | return builder.createOrFold<LLVM::IntToPtrOp>(location: loc, args&: targetType, args&: val); |
| 830 | return builder.createOrFold<LLVM::BitcastOp>(location: loc, args&: targetType, args&: val); |
| 831 | } |
| 832 | |
| 833 | /// Constructs operations that convert `srcValue` into a new value of type |
| 834 | /// `targetType`. Assumes the types have the same bitsize. |
| 835 | static Value castSameSizedTypes(OpBuilder &builder, Location loc, |
| 836 | Value srcValue, Type targetType, |
| 837 | const DataLayout &dataLayout) { |
| 838 | Type srcType = srcValue.getType(); |
| 839 | assert(areConversionCompatible(dataLayout, targetType, srcType, |
| 840 | /*narrowingConversion=*/true) && |
| 841 | "expected that the compatibility was checked before" ); |
| 842 | |
| 843 | // Nothing has to be done if the types are already the same. |
| 844 | if (srcType == targetType) |
| 845 | return srcValue; |
| 846 | |
| 847 | // In the special case of casting one pointer to another, we want to generate |
| 848 | // an address space cast. Bitcasts of pointers are not allowed and using |
| 849 | // pointer to integer conversions are not equivalent due to the loss of |
| 850 | // provenance. |
| 851 | if (isa<LLVM::LLVMPointerType>(Val: targetType) && |
| 852 | isa<LLVM::LLVMPointerType>(Val: srcType)) |
| 853 | return builder.createOrFold<LLVM::AddrSpaceCastOp>(location: loc, args&: targetType, |
| 854 | args&: srcValue); |
| 855 | |
| 856 | // For all other castable types, casting through integers is necessary. |
| 857 | Value replacement = castToSameSizedInt(builder, loc, val: srcValue, dataLayout); |
| 858 | return castIntValueToSameSizedType(builder, loc, val: replacement, targetType); |
| 859 | } |
| 860 | |
| 861 | /// Constructs operations that convert `srcValue` into a new value of type |
| 862 | /// `targetType`. Performs bit-level extraction if the source type is larger |
| 863 | /// than the target type. Assumes that this conversion is possible. |
| 864 | static Value createExtractAndCast(OpBuilder &builder, Location loc, |
| 865 | Value srcValue, Type targetType, |
| 866 | const DataLayout &dataLayout) { |
| 867 | // Get the types of the source and target values. |
| 868 | Type srcType = srcValue.getType(); |
| 869 | assert(areConversionCompatible(dataLayout, targetType, srcType, |
| 870 | /*narrowingConversion=*/true) && |
| 871 | "expected that the compatibility was checked before" ); |
| 872 | |
| 873 | // Nothing has to be done if the types are already the same. This also |
| 874 | // avoids querying the bit size of scalable vector types below. |
| 875 | if (srcType == targetType) |
| 876 | return srcValue; |
| 877 | |
| 878 | uint64_t srcTypeSize = dataLayout.getTypeSizeInBits(t: srcType); |
| 879 | uint64_t targetTypeSize = dataLayout.getTypeSizeInBits(t: targetType); |
| 880 | if (srcTypeSize == targetTypeSize) |
| 881 | return castSameSizedTypes(builder, loc, srcValue, targetType, dataLayout); |
| 882 | |
| 883 | // First, cast the value to a same-sized integer type. |
| 884 | Value replacement = castToSameSizedInt(builder, loc, val: srcValue, dataLayout); |
| 885 | |
| 886 | // Truncate the integer if the size of the target is less than the value. |
| 887 | if (isBigEndian(dataLayout)) { |
| 888 | uint64_t shiftAmount = srcTypeSize - targetTypeSize; |
| 889 | auto shiftConstant = LLVM::ConstantOp::create( |
| 890 | builder, location: loc, value: builder.getIntegerAttr(type: srcType, value: shiftAmount)); |
| 891 | replacement = |
| 892 | builder.createOrFold<LLVM::LShrOp>(location: loc, args&: srcValue, args&: shiftConstant); |
| 893 | } |
| 894 | |
| 895 | replacement = LLVM::TruncOp::create( |
| 896 | builder, location: loc, res: builder.getIntegerType(width: targetTypeSize), arg: replacement); |
| 897 | |
| 898 | // Now cast the integer to the actual target type if required. |
| 899 | return castIntValueToSameSizedType(builder, loc, val: replacement, targetType); |
| 900 | } |
| 901 | |
| 902 | /// Constructs operations that insert the bits of `srcValue` into the |
| 903 | /// "beginning" of `reachingDef` (beginning is endianness dependent). |
| 904 | /// Assumes that this conversion is possible. |
| 905 | static Value createInsertAndCast(OpBuilder &builder, Location loc, |
| 906 | Value srcValue, Value reachingDef, |
| 907 | const DataLayout &dataLayout) { |
| 908 | |
| 909 | assert(areConversionCompatible(dataLayout, reachingDef.getType(), |
| 910 | srcValue.getType(), |
| 911 | /*narrowingConversion=*/false) && |
| 912 | "expected that the compatibility was checked before" ); |
| 913 | |
| 914 | // Nothing has to be done if the types are already the same. This also |
| 915 | // avoids querying the bit size of scalable vector types below. |
| 916 | if (srcValue.getType() == reachingDef.getType()) |
| 917 | return srcValue; |
| 918 | |
| 919 | uint64_t valueTypeSize = dataLayout.getTypeSizeInBits(t: srcValue.getType()); |
| 920 | uint64_t slotTypeSize = dataLayout.getTypeSizeInBits(t: reachingDef.getType()); |
| 921 | if (slotTypeSize == valueTypeSize) |
| 922 | return castSameSizedTypes(builder, loc, srcValue, targetType: reachingDef.getType(), |
| 923 | dataLayout); |
| 924 | |
| 925 | // In the case where the store only overwrites parts of the memory, |
| 926 | // bit fiddling is required to construct the new value. |
| 927 | |
| 928 | // First convert both values to integers of the same size. |
| 929 | Value defAsInt = castToSameSizedInt(builder, loc, val: reachingDef, dataLayout); |
| 930 | Value valueAsInt = castToSameSizedInt(builder, loc, val: srcValue, dataLayout); |
| 931 | // Extend the value to the size of the reaching definition. |
| 932 | valueAsInt = |
| 933 | builder.createOrFold<LLVM::ZExtOp>(location: loc, args: defAsInt.getType(), args&: valueAsInt); |
| 934 | uint64_t sizeDifference = slotTypeSize - valueTypeSize; |
| 935 | if (isBigEndian(dataLayout)) { |
| 936 | // On big endian systems, a store to the base pointer overwrites the most |
| 937 | // significant bits. To accomodate for this, the stored value needs to be |
| 938 | // shifted into the according position. |
| 939 | Value bigEndianShift = LLVM::ConstantOp::create( |
| 940 | builder, location: loc, |
| 941 | value: builder.getIntegerAttr(type: defAsInt.getType(), value: sizeDifference)); |
| 942 | valueAsInt = |
| 943 | builder.createOrFold<LLVM::ShlOp>(location: loc, args&: valueAsInt, args&: bigEndianShift); |
| 944 | } |
| 945 | |
| 946 | // Construct the mask that is used to erase the bits that are overwritten by |
| 947 | // the store. |
| 948 | APInt maskValue; |
| 949 | if (isBigEndian(dataLayout)) { |
| 950 | // Build a mask that has the most significant bits set to zero. |
| 951 | // Note: This is the same as 2^sizeDifference - 1 |
| 952 | maskValue = APInt::getAllOnes(numBits: sizeDifference).zext(width: slotTypeSize); |
| 953 | } else { |
| 954 | // Build a mask that has the least significant bits set to zero. |
| 955 | // Note: This is the same as -(2^valueTypeSize) |
| 956 | maskValue = APInt::getAllOnes(numBits: valueTypeSize).zext(width: slotTypeSize); |
| 957 | maskValue.flipAllBits(); |
| 958 | } |
| 959 | |
| 960 | // Mask out the affected bits ... |
| 961 | Value mask = LLVM::ConstantOp::create( |
| 962 | builder, location: loc, value: builder.getIntegerAttr(type: defAsInt.getType(), value: maskValue)); |
| 963 | Value masked = builder.createOrFold<LLVM::AndOp>(location: loc, args&: defAsInt, args&: mask); |
| 964 | |
| 965 | // ... and combine the result with the new value. |
| 966 | Value combined = builder.createOrFold<LLVM::OrOp>(location: loc, args&: masked, args&: valueAsInt); |
| 967 | |
| 968 | return castIntValueToSameSizedType(builder, loc, val: combined, |
| 969 | targetType: reachingDef.getType()); |
| 970 | } |
| 971 | |
| 972 | Value LLVM::StoreOp::getStored(const MemorySlot &slot, OpBuilder &builder, |
| 973 | Value reachingDef, |
| 974 | const DataLayout &dataLayout) { |
| 975 | assert(reachingDef && reachingDef.getType() == slot.valueType && |
| 976 | "expected the reaching definition's type to match the slot's type" ); |
| 977 | return createInsertAndCast(builder, loc: getLoc(), srcValue: getValue(), reachingDef, |
| 978 | dataLayout); |
| 979 | } |
| 980 | |
| 981 | bool LLVM::LoadOp::canUsesBeRemoved( |
| 982 | const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| 983 | SmallVectorImpl<OpOperand *> &newBlockingUses, |
| 984 | const DataLayout &dataLayout) { |
| 985 | if (blockingUses.size() != 1) |
| 986 | return false; |
| 987 | Value blockingUse = (*blockingUses.begin())->get(); |
| 988 | // If the blocking use is the slot ptr itself, there will be enough |
| 989 | // context to reconstruct the result of the load at removal time, so it can |
| 990 | // be removed (provided it is not volatile). |
| 991 | return blockingUse == slot.ptr && getAddr() == slot.ptr && |
| 992 | areConversionCompatible(layout: dataLayout, targetType: getResult().getType(), |
| 993 | srcType: slot.valueType, |
| 994 | /*narrowingConversion=*/true) && |
| 995 | !getVolatile_(); |
| 996 | } |
| 997 | |
| 998 | DeletionKind LLVM::LoadOp::removeBlockingUses( |
| 999 | const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| 1000 | OpBuilder &builder, Value reachingDefinition, |
| 1001 | const DataLayout &dataLayout) { |
| 1002 | // `canUsesBeRemoved` checked this blocking use must be the loaded slot |
| 1003 | // pointer. |
| 1004 | Value newResult = createExtractAndCast(builder, loc: getLoc(), srcValue: reachingDefinition, |
| 1005 | targetType: getResult().getType(), dataLayout); |
| 1006 | getResult().replaceAllUsesWith(newValue: newResult); |
| 1007 | return DeletionKind::Delete; |
| 1008 | } |
| 1009 | |
| 1010 | bool LLVM::StoreOp::canUsesBeRemoved( |
| 1011 | const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| 1012 | SmallVectorImpl<OpOperand *> &newBlockingUses, |
| 1013 | const DataLayout &dataLayout) { |
| 1014 | if (blockingUses.size() != 1) |
| 1015 | return false; |
| 1016 | Value blockingUse = (*blockingUses.begin())->get(); |
| 1017 | // If the blocking use is the slot ptr itself, dropping the store is |
| 1018 | // fine, provided we are currently promoting its target value. Don't allow a |
| 1019 | // store OF the slot pointer, only INTO the slot pointer. |
| 1020 | return blockingUse == slot.ptr && getAddr() == slot.ptr && |
| 1021 | getValue() != slot.ptr && |
| 1022 | areConversionCompatible(layout: dataLayout, targetType: slot.valueType, |
| 1023 | srcType: getValue().getType(), |
| 1024 | /*narrowingConversion=*/false) && |
| 1025 | !getVolatile_(); |
| 1026 | } |
| 1027 | |
| 1028 | DeletionKind LLVM::StoreOp::removeBlockingUses( |
| 1029 | const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| 1030 | OpBuilder &builder, Value reachingDefinition, |
| 1031 | const DataLayout &dataLayout) { |
| 1032 | return DeletionKind::Delete; |
| 1033 | } |
| 1034 | |
| 1035 | /// Checks if `slot` can be accessed through the provided access type. |
| 1036 | static bool isValidAccessType(const MemorySlot &slot, Type accessType, |
| 1037 | const DataLayout &dataLayout) { |
| 1038 | return dataLayout.getTypeSize(t: accessType) <= |
| 1039 | dataLayout.getTypeSize(t: slot.valueType); |
| 1040 | } |
| 1041 | |
| 1042 | LogicalResult LLVM::LoadOp::ensureOnlySafeAccesses( |
| 1043 | const MemorySlot &slot, SmallVectorImpl<MemorySlot> &mustBeSafelyUsed, |
| 1044 | const DataLayout &dataLayout) { |
| 1045 | return success(IsSuccess: getAddr() != slot.ptr || |
| 1046 | isValidAccessType(slot, accessType: getType(), dataLayout)); |
| 1047 | } |
| 1048 | |
| 1049 | LogicalResult LLVM::StoreOp::ensureOnlySafeAccesses( |
| 1050 | const MemorySlot &slot, SmallVectorImpl<MemorySlot> &mustBeSafelyUsed, |
| 1051 | const DataLayout &dataLayout) { |
| 1052 | return success(IsSuccess: getAddr() != slot.ptr || |
| 1053 | isValidAccessType(slot, accessType: getValue().getType(), dataLayout)); |
| 1054 | } |
| 1055 | |
| 1056 | /// Returns the subslot's type at the requested index. |
| 1057 | static Type getTypeAtIndex(const DestructurableMemorySlot &slot, |
| 1058 | Attribute index) { |
| 1059 | auto subelementIndexMap = |
| 1060 | cast<DestructurableTypeInterface>(Val: slot.valueType).getSubelementIndexMap(); |
| 1061 | if (!subelementIndexMap) |
| 1062 | return {}; |
| 1063 | assert(!subelementIndexMap->empty()); |
| 1064 | |
| 1065 | // Note: Returns a null-type when no entry was found. |
| 1066 | return subelementIndexMap->lookup(Val: index); |
| 1067 | } |
| 1068 | |
| 1069 | bool LLVM::LoadOp::canRewire(const DestructurableMemorySlot &slot, |
| 1070 | SmallPtrSetImpl<Attribute> &usedIndices, |
| 1071 | SmallVectorImpl<MemorySlot> &mustBeSafelyUsed, |
| 1072 | const DataLayout &dataLayout) { |
| 1073 | if (getVolatile_()) |
| 1074 | return false; |
| 1075 | |
| 1076 | // A load always accesses the first element of the destructured slot. |
| 1077 | auto index = IntegerAttr::get(type: IntegerType::get(context: getContext(), width: 32), value: 0); |
| 1078 | Type subslotType = getTypeAtIndex(slot, index); |
| 1079 | if (!subslotType) |
| 1080 | return false; |
| 1081 | |
| 1082 | // The access can only be replaced when the subslot is read within its bounds. |
| 1083 | if (dataLayout.getTypeSize(t: getType()) > dataLayout.getTypeSize(t: subslotType)) |
| 1084 | return false; |
| 1085 | |
| 1086 | usedIndices.insert(Ptr: index); |
| 1087 | return true; |
| 1088 | } |
| 1089 | |
| 1090 | DeletionKind LLVM::LoadOp::rewire(const DestructurableMemorySlot &slot, |
| 1091 | DenseMap<Attribute, MemorySlot> &subslots, |
| 1092 | OpBuilder &builder, |
| 1093 | const DataLayout &dataLayout) { |
| 1094 | auto index = IntegerAttr::get(type: IntegerType::get(context: getContext(), width: 32), value: 0); |
| 1095 | auto it = subslots.find(Val: index); |
| 1096 | assert(it != subslots.end()); |
| 1097 | |
| 1098 | getAddrMutable().set(it->getSecond().ptr); |
| 1099 | return DeletionKind::Keep; |
| 1100 | } |
| 1101 | |
| 1102 | bool LLVM::StoreOp::canRewire(const DestructurableMemorySlot &slot, |
| 1103 | SmallPtrSetImpl<Attribute> &usedIndices, |
| 1104 | SmallVectorImpl<MemorySlot> &mustBeSafelyUsed, |
| 1105 | const DataLayout &dataLayout) { |
| 1106 | if (getVolatile_()) |
| 1107 | return false; |
| 1108 | |
| 1109 | // Storing the pointer to memory cannot be dealt with. |
| 1110 | if (getValue() == slot.ptr) |
| 1111 | return false; |
| 1112 | |
| 1113 | // A store always accesses the first element of the destructured slot. |
| 1114 | auto index = IntegerAttr::get(type: IntegerType::get(context: getContext(), width: 32), value: 0); |
| 1115 | Type subslotType = getTypeAtIndex(slot, index); |
| 1116 | if (!subslotType) |
| 1117 | return false; |
| 1118 | |
| 1119 | // The access can only be replaced when the subslot is read within its bounds. |
| 1120 | if (dataLayout.getTypeSize(t: getValue().getType()) > |
| 1121 | dataLayout.getTypeSize(t: subslotType)) |
| 1122 | return false; |
| 1123 | |
| 1124 | usedIndices.insert(Ptr: index); |
| 1125 | return true; |
| 1126 | } |
| 1127 | |
| 1128 | DeletionKind LLVM::StoreOp::rewire(const DestructurableMemorySlot &slot, |
| 1129 | DenseMap<Attribute, MemorySlot> &subslots, |
| 1130 | OpBuilder &builder, |
| 1131 | const DataLayout &dataLayout) { |
| 1132 | auto index = IntegerAttr::get(type: IntegerType::get(context: getContext(), width: 32), value: 0); |
| 1133 | auto it = subslots.find(Val: index); |
| 1134 | assert(it != subslots.end()); |
| 1135 | |
| 1136 | getAddrMutable().set(it->getSecond().ptr); |
| 1137 | return DeletionKind::Keep; |
| 1138 | } |
| 1139 | |
| 1140 | //===----------------------------------------------------------------------===// |
| 1141 | // Interfaces for discardable OPs |
| 1142 | //===----------------------------------------------------------------------===// |
| 1143 | |
| 1144 | /// Conditions the deletion of the operation to the removal of all its uses. |
| 1145 | bool mlir::LLVM::forwardToUsers(Operation *op, |
| 1146 | SmallVectorImpl<OpOperand *> &newBlockingUses) { |
| 1147 | for (Value result : op->getResults()) |
| 1148 | for (OpOperand &use : result.getUses()) |
| 1149 | newBlockingUses.push_back(Elt: &use); |
| 1150 | return true; |
| 1151 | } |
| 1152 | |
| 1153 | //===----------------------------------------------------------------------===// |
| 1154 | // Interfaces for GEPOp |
| 1155 | //===----------------------------------------------------------------------===// |
| 1156 | |
| 1157 | static bool hasAllZeroIndices(LLVM::GEPOp gepOp) { |
| 1158 | return llvm::all_of(Range: gepOp.getIndices(), P: [](auto index) { |
| 1159 | auto indexAttr = llvm::dyn_cast_if_present<IntegerAttr>(index); |
| 1160 | return indexAttr && indexAttr.getValue() == 0; |
| 1161 | }); |
| 1162 | } |
| 1163 | |
| 1164 | bool LLVM::GEPOp::canUsesBeRemoved( |
| 1165 | const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| 1166 | SmallVectorImpl<OpOperand *> &newBlockingUses, |
| 1167 | const DataLayout &dataLayout) { |
| 1168 | // GEP can be removed as long as it is a no-op and its users can be removed. |
| 1169 | // `inrange` is only valid on constant GEP expressions, so an inrange GEP on |
| 1170 | // an alloca is illegal and we bail out. |
| 1171 | if (getInrangeAttr() || !hasAllZeroIndices(gepOp: *this)) |
| 1172 | return false; |
| 1173 | return forwardToUsers(op: *this, newBlockingUses); |
| 1174 | } |
| 1175 | |
| 1176 | DeletionKind LLVM::GEPOp::removeBlockingUses( |
| 1177 | const SmallPtrSetImpl<OpOperand *> &blockingUses, OpBuilder &builder) { |
| 1178 | return DeletionKind::Delete; |
| 1179 | } |
| 1180 | |
| 1181 | /// Returns the amount of bytes the provided GEP elements will offset the |
| 1182 | /// pointer by. Returns nullopt if no constant offset could be computed. |
| 1183 | static std::optional<uint64_t> gepToByteOffset(const DataLayout &dataLayout, |
| 1184 | LLVM::GEPOp gep) { |
| 1185 | // Collects all indices. |
| 1186 | SmallVector<uint64_t> indices; |
| 1187 | for (auto index : gep.getIndices()) { |
| 1188 | auto constIndex = dyn_cast<IntegerAttr>(Val&: index); |
| 1189 | if (!constIndex) |
| 1190 | return {}; |
| 1191 | int64_t gepIndex = constIndex.getInt(); |
| 1192 | // Negative indices are not supported. |
| 1193 | if (gepIndex < 0) |
| 1194 | return {}; |
| 1195 | indices.push_back(Elt: gepIndex); |
| 1196 | } |
| 1197 | |
| 1198 | Type currentType = gep.getElemType(); |
| 1199 | uint64_t offset = indices[0] * dataLayout.getTypeSize(t: currentType); |
| 1200 | |
| 1201 | for (uint64_t index : llvm::drop_begin(RangeOrContainer&: indices)) { |
| 1202 | bool shouldCancel = |
| 1203 | TypeSwitch<Type, bool>(currentType) |
| 1204 | .Case(caseFn: [&](LLVM::LLVMArrayType arrayType) { |
| 1205 | offset += |
| 1206 | index * dataLayout.getTypeSize(t: arrayType.getElementType()); |
| 1207 | currentType = arrayType.getElementType(); |
| 1208 | return false; |
| 1209 | }) |
| 1210 | .Case(caseFn: [&](LLVM::LLVMStructType structType) { |
| 1211 | offset += structType.getElementOffset(dataLayout, index); |
| 1212 | currentType = structType.getBody()[index]; |
| 1213 | return false; |
| 1214 | }) |
| 1215 | .Default(defaultFn: [&](Type type) { |
| 1216 | LDBG() << "[sroa] Unsupported type for offset computations" |
| 1217 | << type; |
| 1218 | return true; |
| 1219 | }); |
| 1220 | |
| 1221 | if (shouldCancel) |
| 1222 | return std::nullopt; |
| 1223 | } |
| 1224 | |
| 1225 | return offset; |
| 1226 | } |
| 1227 | |
| 1228 | namespace { |
| 1229 | /// A struct that stores both the index into the aggregate type of the slot as |
| 1230 | /// well as the corresponding byte offset in memory. |
| 1231 | struct SubslotAccessInfo { |
| 1232 | /// The parent slot's index that the access falls into. |
| 1233 | uint32_t index; |
| 1234 | /// The offset into the subslot of the access. |
| 1235 | uint64_t subslotOffset; |
| 1236 | }; |
| 1237 | } // namespace |
| 1238 | |
| 1239 | /// Computes subslot access information for an access into `slot` with the given |
| 1240 | /// offset. |
| 1241 | /// Returns nullopt when the offset is out-of-bounds or when the access is into |
| 1242 | /// the padding of `slot`. |
| 1243 | static std::optional<SubslotAccessInfo> |
| 1244 | getSubslotAccessInfo(const DestructurableMemorySlot &slot, |
| 1245 | const DataLayout &dataLayout, LLVM::GEPOp gep) { |
| 1246 | std::optional<uint64_t> offset = gepToByteOffset(dataLayout, gep); |
| 1247 | if (!offset) |
| 1248 | return {}; |
| 1249 | |
| 1250 | // Helper to check that a constant index is in the bounds of the GEP index |
| 1251 | // representation. LLVM dialects's GEP arguments have a limited bitwidth, thus |
| 1252 | // this additional check is necessary. |
| 1253 | auto isOutOfBoundsGEPIndex = [](uint64_t index) { |
| 1254 | return index >= (1 << LLVM::kGEPConstantBitWidth); |
| 1255 | }; |
| 1256 | |
| 1257 | Type type = slot.valueType; |
| 1258 | if (*offset >= dataLayout.getTypeSize(t: type)) |
| 1259 | return {}; |
| 1260 | return TypeSwitch<Type, std::optional<SubslotAccessInfo>>(type) |
| 1261 | .Case(caseFn: [&](LLVM::LLVMArrayType arrayType) |
| 1262 | -> std::optional<SubslotAccessInfo> { |
| 1263 | // Find which element of the array contains the offset. |
| 1264 | uint64_t elemSize = dataLayout.getTypeSize(t: arrayType.getElementType()); |
| 1265 | uint64_t index = *offset / elemSize; |
| 1266 | if (isOutOfBoundsGEPIndex(index)) |
| 1267 | return {}; |
| 1268 | return SubslotAccessInfo{.index: static_cast<uint32_t>(index), |
| 1269 | .subslotOffset: *offset - (index * elemSize)}; |
| 1270 | }) |
| 1271 | .Case(caseFn: [&](LLVM::LLVMStructType structType) |
| 1272 | -> std::optional<SubslotAccessInfo> { |
| 1273 | // Walk over the elements of the struct to find in which of |
| 1274 | // them the offset is. |
| 1275 | for (auto [index, elem] : llvm::enumerate(First: structType.getBody())) { |
| 1276 | uint64_t elemOffset = structType.getElementOffset(dataLayout, index); |
| 1277 | // If the offset is in padding, cancel the rewrite. |
| 1278 | if (*offset < elemOffset) |
| 1279 | return {}; |
| 1280 | |
| 1281 | if (*offset < elemOffset + dataLayout.getTypeSize(t: elem)) { |
| 1282 | if (isOutOfBoundsGEPIndex(index)) |
| 1283 | return {}; |
| 1284 | // The offset is within this element, stop iterating the |
| 1285 | // struct and return the index. |
| 1286 | return SubslotAccessInfo{.index: static_cast<uint32_t>(index), |
| 1287 | .subslotOffset: *offset - elemOffset}; |
| 1288 | } |
| 1289 | } |
| 1290 | |
| 1291 | return {}; |
| 1292 | }); |
| 1293 | } |
| 1294 | |
| 1295 | /// Constructs a byte array type of the given size. |
| 1296 | static LLVM::LLVMArrayType getByteArrayType(MLIRContext *context, |
| 1297 | unsigned size) { |
| 1298 | auto byteType = IntegerType::get(context, width: 8); |
| 1299 | return LLVM::LLVMArrayType::get(context, elementType: byteType, numElements: size); |
| 1300 | } |
| 1301 | |
| 1302 | LogicalResult LLVM::GEPOp::ensureOnlySafeAccesses( |
| 1303 | const MemorySlot &slot, SmallVectorImpl<MemorySlot> &mustBeSafelyUsed, |
| 1304 | const DataLayout &dataLayout) { |
| 1305 | if (getBase() != slot.ptr) |
| 1306 | return success(); |
| 1307 | std::optional<uint64_t> gepOffset = gepToByteOffset(dataLayout, gep: *this); |
| 1308 | if (!gepOffset) |
| 1309 | return failure(); |
| 1310 | uint64_t slotSize = dataLayout.getTypeSize(t: slot.valueType); |
| 1311 | // Check that the access is strictly inside the slot. |
| 1312 | if (*gepOffset >= slotSize) |
| 1313 | return failure(); |
| 1314 | // Every access that remains in bounds of the remaining slot is considered |
| 1315 | // legal. |
| 1316 | mustBeSafelyUsed.emplace_back<MemorySlot>( |
| 1317 | Args: {.ptr: getRes(), .valueType: getByteArrayType(context: getContext(), size: slotSize - *gepOffset)}); |
| 1318 | return success(); |
| 1319 | } |
| 1320 | |
| 1321 | bool LLVM::GEPOp::canRewire(const DestructurableMemorySlot &slot, |
| 1322 | SmallPtrSetImpl<Attribute> &usedIndices, |
| 1323 | SmallVectorImpl<MemorySlot> &mustBeSafelyUsed, |
| 1324 | const DataLayout &dataLayout) { |
| 1325 | if (!isa<LLVM::LLVMPointerType>(Val: getBase().getType())) |
| 1326 | return false; |
| 1327 | |
| 1328 | if (getBase() != slot.ptr) |
| 1329 | return false; |
| 1330 | // `inrange` is only valid on constant GEP expressions, so an inrange GEP on |
| 1331 | // an alloca is illegal and SROA bails out. |
| 1332 | if (getInrangeAttr()) |
| 1333 | return false; |
| 1334 | std::optional<SubslotAccessInfo> accessInfo = |
| 1335 | getSubslotAccessInfo(slot, dataLayout, gep: *this); |
| 1336 | if (!accessInfo) |
| 1337 | return false; |
| 1338 | auto indexAttr = |
| 1339 | IntegerAttr::get(type: IntegerType::get(context: getContext(), width: 32), value: accessInfo->index); |
| 1340 | assert(slot.subelementTypes.contains(indexAttr)); |
| 1341 | usedIndices.insert(Ptr: indexAttr); |
| 1342 | |
| 1343 | // The remainder of the subslot should be accesses in-bounds. Thus, we create |
| 1344 | // a dummy slot with the size of the remainder. |
| 1345 | Type subslotType = slot.subelementTypes.lookup(Val: indexAttr); |
| 1346 | uint64_t slotSize = dataLayout.getTypeSize(t: subslotType); |
| 1347 | LLVM::LLVMArrayType remainingSlotType = |
| 1348 | getByteArrayType(context: getContext(), size: slotSize - accessInfo->subslotOffset); |
| 1349 | mustBeSafelyUsed.emplace_back<MemorySlot>(Args: {.ptr: getRes(), .valueType: remainingSlotType}); |
| 1350 | |
| 1351 | return true; |
| 1352 | } |
| 1353 | |
| 1354 | DeletionKind LLVM::GEPOp::rewire(const DestructurableMemorySlot &slot, |
| 1355 | DenseMap<Attribute, MemorySlot> &subslots, |
| 1356 | OpBuilder &builder, |
| 1357 | const DataLayout &dataLayout) { |
| 1358 | std::optional<SubslotAccessInfo> accessInfo = |
| 1359 | getSubslotAccessInfo(slot, dataLayout, gep: *this); |
| 1360 | assert(accessInfo && "expected access info to be checked before" ); |
| 1361 | auto indexAttr = |
| 1362 | IntegerAttr::get(type: IntegerType::get(context: getContext(), width: 32), value: accessInfo->index); |
| 1363 | const MemorySlot &newSlot = subslots.at(Val: indexAttr); |
| 1364 | |
| 1365 | auto byteType = IntegerType::get(context: builder.getContext(), width: 8); |
| 1366 | auto newPtr = builder.createOrFold<LLVM::GEPOp>( |
| 1367 | location: getLoc(), args: getResult().getType(), args&: byteType, args: newSlot.ptr, |
| 1368 | args: ArrayRef<GEPArg>(accessInfo->subslotOffset), args: getNoWrapFlags()); |
| 1369 | getResult().replaceAllUsesWith(newValue: newPtr); |
| 1370 | return DeletionKind::Delete; |
| 1371 | } |
| 1372 | |
| 1373 | SmallVector<Value> mlir::LLVM::AtomicCmpXchgOp::getAccessedOperands() { |
| 1374 | return {getPtr()}; |
| 1375 | } |
| 1376 | |
| 1377 | SmallVector<Value> mlir::LLVM::AtomicRMWOp::getAccessedOperands() { |
| 1378 | return {getPtr()}; |
| 1379 | } |
| 1380 | |
| 1381 | SmallVector<Value> mlir::LLVM::LoadOp::getAccessedOperands() { |
| 1382 | return {getAddr()}; |
| 1383 | } |
| 1384 | |
| 1385 | SmallVector<Value> mlir::LLVM::StoreOp::getAccessedOperands() { |
| 1386 | return {getAddr()}; |
| 1387 | } |
| 1388 | |
| 1389 | #define GET_OP_CLASSES |
| 1390 | #include "mlir/Dialect/LLVMIR/LLVMMemoryOps.cpp.inc" |
| 1391 | |