| 1 | //===- LLVMMemoryIntrinsicOps.cpp - LLVM dialect memory intrinsics --------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file defines the ops of op partition `MemoryIntrinsic` of the LLVM |
| 10 | // dialect. |
| 11 | // |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #include "mlir/Dialect/LLVMIR/LLVMMemoryIntrinsicOps.h" |
| 15 | |
| 16 | #include "IR/LLVMOps.h" |
| 17 | #include "mlir/Dialect/LLVMIR/LLVMArithOps.h" |
| 18 | #include "mlir/Dialect/LLVMIR/LLVMConstantOps.h" |
| 19 | #include "mlir/Dialect/LLVMIR/LLVMConversionOps.h" |
| 20 | #include "mlir/Dialect/LLVMIR/LLVMMemoryOps.h" |
| 21 | #include "mlir/IR/Builders.h" |
| 22 | #include "mlir/IR/DialectImplementation.h" |
| 23 | #include "mlir/IR/Matchers.h" |
| 24 | #include "mlir/Interfaces/DataLayoutInterfaces.h" |
| 25 | #include "mlir/Interfaces/MemorySlotInterfaces.h" |
| 26 | #include "llvm/ADT/STLExtras.h" |
| 27 | #include "llvm/ADT/TypeSwitch.h" |
| 28 | |
| 29 | #include <optional> |
| 30 | |
| 31 | using namespace mlir; |
| 32 | using namespace mlir::LLVM; |
| 33 | |
| 34 | bool LLVM::LifetimeStartOp::canUsesBeRemoved( |
| 35 | const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| 36 | SmallVectorImpl<OpOperand *> &newBlockingUses, |
| 37 | const DataLayout &dataLayout) { |
| 38 | return true; |
| 39 | } |
| 40 | |
| 41 | DeletionKind LLVM::LifetimeStartOp::removeBlockingUses( |
| 42 | const SmallPtrSetImpl<OpOperand *> &blockingUses, OpBuilder &builder) { |
| 43 | return DeletionKind::Delete; |
| 44 | } |
| 45 | |
| 46 | bool LLVM::LifetimeEndOp::canUsesBeRemoved( |
| 47 | const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| 48 | SmallVectorImpl<OpOperand *> &newBlockingUses, |
| 49 | const DataLayout &dataLayout) { |
| 50 | return true; |
| 51 | } |
| 52 | |
| 53 | DeletionKind LLVM::LifetimeEndOp::removeBlockingUses( |
| 54 | const SmallPtrSetImpl<OpOperand *> &blockingUses, OpBuilder &builder) { |
| 55 | return DeletionKind::Delete; |
| 56 | } |
| 57 | |
| 58 | bool LLVM::InvariantStartOp::canUsesBeRemoved( |
| 59 | const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| 60 | SmallVectorImpl<OpOperand *> &newBlockingUses, |
| 61 | const DataLayout &dataLayout) { |
| 62 | return true; |
| 63 | } |
| 64 | |
| 65 | DeletionKind LLVM::InvariantStartOp::removeBlockingUses( |
| 66 | const SmallPtrSetImpl<OpOperand *> &blockingUses, OpBuilder &builder) { |
| 67 | return DeletionKind::Delete; |
| 68 | } |
| 69 | |
| 70 | bool LLVM::InvariantEndOp::canUsesBeRemoved( |
| 71 | const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| 72 | SmallVectorImpl<OpOperand *> &newBlockingUses, |
| 73 | const DataLayout &dataLayout) { |
| 74 | return true; |
| 75 | } |
| 76 | |
| 77 | DeletionKind LLVM::InvariantEndOp::removeBlockingUses( |
| 78 | const SmallPtrSetImpl<OpOperand *> &blockingUses, OpBuilder &builder) { |
| 79 | return DeletionKind::Delete; |
| 80 | } |
| 81 | |
| 82 | bool LLVM::LaunderInvariantGroupOp::canUsesBeRemoved( |
| 83 | const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| 84 | SmallVectorImpl<OpOperand *> &newBlockingUses, |
| 85 | const DataLayout &dataLayout) { |
| 86 | return forwardToUsers(op: *this, newBlockingUses); |
| 87 | } |
| 88 | |
| 89 | DeletionKind LLVM::LaunderInvariantGroupOp::removeBlockingUses( |
| 90 | const SmallPtrSetImpl<OpOperand *> &blockingUses, OpBuilder &builder) { |
| 91 | return DeletionKind::Delete; |
| 92 | } |
| 93 | |
| 94 | //===----------------------------------------------------------------------===// |
| 95 | // Utilities for memory intrinsics |
| 96 | //===----------------------------------------------------------------------===// |
| 97 | |
| 98 | namespace { |
| 99 | /// Returns the length of the given memory intrinsic in bytes if it can be known |
| 100 | /// at compile-time on a best-effort basis, nothing otherwise. |
| 101 | template <class MemIntr> |
| 102 | std::optional<uint64_t> getStaticMemIntrLen(MemIntr op) { |
| 103 | APInt memIntrLen; |
| 104 | if (!matchPattern(op.getLen(), m_ConstantInt(bind_value: &memIntrLen))) |
| 105 | return {}; |
| 106 | if (memIntrLen.getBitWidth() > 64) |
| 107 | return {}; |
| 108 | return memIntrLen.getZExtValue(); |
| 109 | } |
| 110 | |
| 111 | /// Returns the length of the given memory intrinsic in bytes if it can be known |
| 112 | /// at compile-time on a best-effort basis, nothing otherwise. |
| 113 | /// Because MemcpyInlineOp has its length encoded as an attribute, this requires |
| 114 | /// specialized handling. |
| 115 | template <> |
| 116 | std::optional<uint64_t> getStaticMemIntrLen(LLVM::MemcpyInlineOp op) { |
| 117 | APInt memIntrLen = op.getLen(); |
| 118 | if (memIntrLen.getBitWidth() > 64) |
| 119 | return {}; |
| 120 | return memIntrLen.getZExtValue(); |
| 121 | } |
| 122 | |
| 123 | /// Returns the length of the given memory intrinsic in bytes if it can be known |
| 124 | /// at compile-time on a best-effort basis, nothing otherwise. |
| 125 | /// Because MemsetInlineOp has its length encoded as an attribute, this requires |
| 126 | /// specialized handling. |
| 127 | template <> |
| 128 | std::optional<uint64_t> getStaticMemIntrLen(LLVM::MemsetInlineOp op) { |
| 129 | APInt memIntrLen = op.getLen(); |
| 130 | if (memIntrLen.getBitWidth() > 64) |
| 131 | return {}; |
| 132 | return memIntrLen.getZExtValue(); |
| 133 | } |
| 134 | |
| 135 | /// Returns an integer attribute representing the length of a memset intrinsic |
| 136 | template <class MemsetIntr> |
| 137 | IntegerAttr createMemsetLenAttr(MemsetIntr op) { |
| 138 | IntegerAttr memsetLenAttr; |
| 139 | bool successfulMatch = |
| 140 | matchPattern(op.getLen(), m_Constant<IntegerAttr>(bind_value: &memsetLenAttr)); |
| 141 | (void)successfulMatch; |
| 142 | assert(successfulMatch); |
| 143 | return memsetLenAttr; |
| 144 | } |
| 145 | |
| 146 | /// Returns an integer attribute representing the length of a memset intrinsic |
| 147 | /// Because MemsetInlineOp has its length encoded as an attribute, this requires |
| 148 | /// specialized handling. |
| 149 | template <> |
| 150 | IntegerAttr createMemsetLenAttr(LLVM::MemsetInlineOp op) { |
| 151 | return op.getLenAttr(); |
| 152 | } |
| 153 | |
| 154 | /// Creates a memset intrinsic of that matches the `toReplace` intrinsic |
| 155 | /// using the provided parameters. There are template specializations for |
| 156 | /// MemsetOp and MemsetInlineOp. |
| 157 | template <class MemsetIntr> |
| 158 | void createMemsetIntr(OpBuilder &builder, MemsetIntr toReplace, |
| 159 | IntegerAttr memsetLenAttr, uint64_t newMemsetSize, |
| 160 | DenseMap<Attribute, MemorySlot> &subslots, |
| 161 | Attribute index); |
| 162 | |
| 163 | template <> |
| 164 | void createMemsetIntr(OpBuilder &builder, LLVM::MemsetOp toReplace, |
| 165 | IntegerAttr memsetLenAttr, uint64_t newMemsetSize, |
| 166 | DenseMap<Attribute, MemorySlot> &subslots, |
| 167 | Attribute index) { |
| 168 | Value newMemsetSizeValue = |
| 169 | LLVM::ConstantOp::create( |
| 170 | builder, location: toReplace.getLen().getLoc(), |
| 171 | value: IntegerAttr::get(type: memsetLenAttr.getType(), value: newMemsetSize)) |
| 172 | .getResult(); |
| 173 | |
| 174 | LLVM::MemsetOp::create(builder, location: toReplace.getLoc(), dst: subslots.at(Val: index).ptr, |
| 175 | val: toReplace.getVal(), len: newMemsetSizeValue, |
| 176 | isVolatile: toReplace.getIsVolatile()); |
| 177 | } |
| 178 | |
| 179 | template <> |
| 180 | void createMemsetIntr(OpBuilder &builder, LLVM::MemsetInlineOp toReplace, |
| 181 | IntegerAttr memsetLenAttr, uint64_t newMemsetSize, |
| 182 | DenseMap<Attribute, MemorySlot> &subslots, |
| 183 | Attribute index) { |
| 184 | auto newMemsetSizeValue = |
| 185 | IntegerAttr::get(type: memsetLenAttr.getType(), value: newMemsetSize); |
| 186 | |
| 187 | LLVM::MemsetInlineOp::create(builder, location: toReplace.getLoc(), |
| 188 | dst: subslots.at(Val: index).ptr, val: toReplace.getVal(), |
| 189 | len: newMemsetSizeValue, isVolatile: toReplace.getIsVolatile()); |
| 190 | } |
| 191 | } // namespace |
| 192 | |
| 193 | /// Returns whether one can be sure the memory intrinsic does not write outside |
| 194 | /// of the bounds of the given slot, on a best-effort basis. |
| 195 | template <class MemIntr> |
| 196 | static bool definitelyWritesOnlyWithinSlot(MemIntr op, const MemorySlot &slot, |
| 197 | const DataLayout &dataLayout) { |
| 198 | if (!isa<LLVM::LLVMPointerType>(Val: slot.ptr.getType()) || |
| 199 | op.getDst() != slot.ptr) |
| 200 | return false; |
| 201 | |
| 202 | std::optional<uint64_t> memIntrLen = getStaticMemIntrLen(op); |
| 203 | return memIntrLen && *memIntrLen <= dataLayout.getTypeSize(t: slot.valueType); |
| 204 | } |
| 205 | |
| 206 | /// Checks whether all indices are i32. This is used to check GEPs can index |
| 207 | /// into them. |
| 208 | static bool areAllIndicesI32(const DestructurableMemorySlot &slot) { |
| 209 | Type i32 = IntegerType::get(context: slot.ptr.getContext(), width: 32); |
| 210 | return llvm::all_of(Range: llvm::make_first_range(c: slot.subelementTypes), |
| 211 | P: [&](Attribute index) { |
| 212 | auto intIndex = dyn_cast<IntegerAttr>(Val&: index); |
| 213 | return intIndex && intIndex.getType() == i32; |
| 214 | }); |
| 215 | } |
| 216 | |
| 217 | //===----------------------------------------------------------------------===// |
| 218 | // Interfaces for memset and memset.inline |
| 219 | //===----------------------------------------------------------------------===// |
| 220 | |
| 221 | template <class MemsetIntr> |
| 222 | static bool memsetCanRewire(MemsetIntr op, const DestructurableMemorySlot &slot, |
| 223 | SmallPtrSetImpl<Attribute> &usedIndices, |
| 224 | SmallVectorImpl<MemorySlot> &mustBeSafelyUsed, |
| 225 | const DataLayout &dataLayout) { |
| 226 | if (&slot.valueType.getDialect() != op.getOperation()->getDialect()) |
| 227 | return false; |
| 228 | |
| 229 | if (op.getIsVolatile()) |
| 230 | return false; |
| 231 | |
| 232 | if (!cast<DestructurableTypeInterface>(Val: slot.valueType) |
| 233 | .getSubelementIndexMap()) |
| 234 | return false; |
| 235 | |
| 236 | if (!areAllIndicesI32(slot)) |
| 237 | return false; |
| 238 | |
| 239 | return definitelyWritesOnlyWithinSlot(op, slot, dataLayout); |
| 240 | } |
| 241 | |
| 242 | template <class MemsetIntr> |
| 243 | static Value memsetGetStored(MemsetIntr op, const MemorySlot &slot, |
| 244 | OpBuilder &builder) { |
| 245 | /// Returns an integer value that is `width` bits wide representing the value |
| 246 | /// assigned to the slot by memset. |
| 247 | auto buildMemsetValue = [&](unsigned width) -> Value { |
| 248 | assert(width % 8 == 0); |
| 249 | auto intType = IntegerType::get(context: op.getContext(), width); |
| 250 | |
| 251 | // If we know the pattern at compile time, we can compute and assign a |
| 252 | // constant directly. |
| 253 | IntegerAttr constantPattern; |
| 254 | if (matchPattern(op.getVal(), m_Constant(bind_value: &constantPattern))) { |
| 255 | assert(constantPattern.getValue().getBitWidth() == 8); |
| 256 | APInt memsetVal(/*numBits=*/width, /*val=*/0); |
| 257 | for (unsigned loBit = 0; loBit < width; loBit += 8) |
| 258 | memsetVal.insertBits(SubBits: constantPattern.getValue(), bitPosition: loBit); |
| 259 | return LLVM::ConstantOp::create(builder, op.getLoc(), |
| 260 | IntegerAttr::get(intType, memsetVal)); |
| 261 | } |
| 262 | |
| 263 | // If the output is a single byte, we can return the pattern directly. |
| 264 | if (width == 8) |
| 265 | return op.getVal(); |
| 266 | |
| 267 | // Otherwise build the memset integer at runtime by repeatedly shifting the |
| 268 | // value and or-ing it with the previous value. |
| 269 | uint64_t coveredBits = 8; |
| 270 | Value currentValue = |
| 271 | LLVM::ZExtOp::create(builder, op.getLoc(), intType, op.getVal()); |
| 272 | while (coveredBits < width) { |
| 273 | Value shiftBy = |
| 274 | LLVM::ConstantOp::create(builder, op.getLoc(), intType, coveredBits); |
| 275 | Value shifted = |
| 276 | LLVM::ShlOp::create(builder, op.getLoc(), currentValue, shiftBy); |
| 277 | currentValue = |
| 278 | LLVM::OrOp::create(builder, op.getLoc(), currentValue, shifted); |
| 279 | coveredBits *= 2; |
| 280 | } |
| 281 | |
| 282 | return currentValue; |
| 283 | }; |
| 284 | return TypeSwitch<Type, Value>(slot.valueType) |
| 285 | .Case([&](IntegerType type) -> Value { |
| 286 | return buildMemsetValue(type.getWidth()); |
| 287 | }) |
| 288 | .Case([&](FloatType type) -> Value { |
| 289 | Value intVal = buildMemsetValue(type.getWidth()); |
| 290 | return LLVM::BitcastOp::create(builder, op.getLoc(), type, intVal); |
| 291 | }) |
| 292 | .DefaultUnreachable( |
| 293 | "getStored should not be called on memset to unsupported type" ); |
| 294 | } |
| 295 | |
| 296 | template <class MemsetIntr> |
| 297 | static bool |
| 298 | memsetCanUsesBeRemoved(MemsetIntr op, const MemorySlot &slot, |
| 299 | const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| 300 | SmallVectorImpl<OpOperand *> &newBlockingUses, |
| 301 | const DataLayout &dataLayout) { |
| 302 | bool canConvertType = |
| 303 | TypeSwitch<Type, bool>(slot.valueType) |
| 304 | .Case<IntegerType, FloatType>([](auto type) { |
| 305 | return type.getWidth() % 8 == 0 && type.getWidth() > 0; |
| 306 | }) |
| 307 | .Default(false); |
| 308 | if (!canConvertType) |
| 309 | return false; |
| 310 | |
| 311 | if (op.getIsVolatile()) |
| 312 | return false; |
| 313 | |
| 314 | return getStaticMemIntrLen(op) == dataLayout.getTypeSize(t: slot.valueType); |
| 315 | } |
| 316 | |
| 317 | template <class MemsetIntr> |
| 318 | static DeletionKind |
| 319 | memsetRewire(MemsetIntr op, const DestructurableMemorySlot &slot, |
| 320 | DenseMap<Attribute, MemorySlot> &subslots, OpBuilder &builder, |
| 321 | const DataLayout &dataLayout) { |
| 322 | |
| 323 | std::optional<DenseMap<Attribute, Type>> types = |
| 324 | cast<DestructurableTypeInterface>(Val: slot.valueType).getSubelementIndexMap(); |
| 325 | |
| 326 | IntegerAttr memsetLenAttr = createMemsetLenAttr(op); |
| 327 | |
| 328 | auto structType = dyn_cast<LLVM::LLVMStructType>(Val: slot.valueType); |
| 329 | |
| 330 | Type i32 = IntegerType::get(context: op.getContext(), width: 32); |
| 331 | uint64_t memsetLen = memsetLenAttr.getValue().getZExtValue(); |
| 332 | for (size_t i = 0; i < types->size(); i++) { |
| 333 | // Create indices on the fly to get elements in the right order. |
| 334 | Attribute index = IntegerAttr::get(type: i32, value: i); |
| 335 | Type elemType = types->at(Val: index); |
| 336 | uint64_t typeSize = dataLayout.getTypeSize(t: elemType); |
| 337 | uint64_t offset = structType ? structType.getElementOffset(dataLayout, index: i) |
| 338 | : i * dataLayout.getTypeAllocSize(t: elemType); |
| 339 | if (offset >= memsetLen) |
| 340 | break; |
| 341 | |
| 342 | // If this subslot is used, apply a new memset to it. |
| 343 | if (subslots.contains(Val: index)) { |
| 344 | uint64_t newMemsetSize = std::min(a: memsetLen - offset, b: typeSize); |
| 345 | createMemsetIntr(builder, op, memsetLenAttr, newMemsetSize, subslots, |
| 346 | index); |
| 347 | } |
| 348 | } |
| 349 | |
| 350 | return DeletionKind::Delete; |
| 351 | } |
| 352 | |
| 353 | bool LLVM::MemsetOp::loadsFrom(const MemorySlot &slot) { return false; } |
| 354 | |
| 355 | bool LLVM::MemsetOp::storesTo(const MemorySlot &slot) { |
| 356 | return getDst() == slot.ptr; |
| 357 | } |
| 358 | |
| 359 | Value LLVM::MemsetOp::getStored(const MemorySlot &slot, OpBuilder &builder, |
| 360 | Value reachingDef, |
| 361 | const DataLayout &dataLayout) { |
| 362 | return memsetGetStored(op: *this, slot, builder); |
| 363 | } |
| 364 | |
| 365 | bool LLVM::MemsetOp::canUsesBeRemoved( |
| 366 | const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| 367 | SmallVectorImpl<OpOperand *> &newBlockingUses, |
| 368 | const DataLayout &dataLayout) { |
| 369 | return memsetCanUsesBeRemoved(op: *this, slot, blockingUses, newBlockingUses, |
| 370 | dataLayout); |
| 371 | } |
| 372 | |
| 373 | DeletionKind LLVM::MemsetOp::removeBlockingUses( |
| 374 | const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| 375 | OpBuilder &builder, Value reachingDefinition, |
| 376 | const DataLayout &dataLayout) { |
| 377 | return DeletionKind::Delete; |
| 378 | } |
| 379 | |
| 380 | LogicalResult LLVM::MemsetOp::ensureOnlySafeAccesses( |
| 381 | const MemorySlot &slot, SmallVectorImpl<MemorySlot> &mustBeSafelyUsed, |
| 382 | const DataLayout &dataLayout) { |
| 383 | return success(IsSuccess: definitelyWritesOnlyWithinSlot(op: *this, slot, dataLayout)); |
| 384 | } |
| 385 | |
| 386 | bool LLVM::MemsetOp::canRewire(const DestructurableMemorySlot &slot, |
| 387 | SmallPtrSetImpl<Attribute> &usedIndices, |
| 388 | SmallVectorImpl<MemorySlot> &mustBeSafelyUsed, |
| 389 | const DataLayout &dataLayout) { |
| 390 | return memsetCanRewire(op: *this, slot, usedIndices, mustBeSafelyUsed, |
| 391 | dataLayout); |
| 392 | } |
| 393 | |
| 394 | DeletionKind LLVM::MemsetOp::rewire(const DestructurableMemorySlot &slot, |
| 395 | DenseMap<Attribute, MemorySlot> &subslots, |
| 396 | OpBuilder &builder, |
| 397 | const DataLayout &dataLayout) { |
| 398 | return memsetRewire(op: *this, slot, subslots, builder, dataLayout); |
| 399 | } |
| 400 | |
| 401 | bool LLVM::MemsetInlineOp::loadsFrom(const MemorySlot &slot) { return false; } |
| 402 | |
| 403 | bool LLVM::MemsetInlineOp::storesTo(const MemorySlot &slot) { |
| 404 | return getDst() == slot.ptr; |
| 405 | } |
| 406 | |
| 407 | Value LLVM::MemsetInlineOp::getStored(const MemorySlot &slot, |
| 408 | OpBuilder &builder, Value reachingDef, |
| 409 | const DataLayout &dataLayout) { |
| 410 | return memsetGetStored(op: *this, slot, builder); |
| 411 | } |
| 412 | |
| 413 | bool LLVM::MemsetInlineOp::canUsesBeRemoved( |
| 414 | const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| 415 | SmallVectorImpl<OpOperand *> &newBlockingUses, |
| 416 | const DataLayout &dataLayout) { |
| 417 | return memsetCanUsesBeRemoved(op: *this, slot, blockingUses, newBlockingUses, |
| 418 | dataLayout); |
| 419 | } |
| 420 | |
| 421 | DeletionKind LLVM::MemsetInlineOp::removeBlockingUses( |
| 422 | const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| 423 | OpBuilder &builder, Value reachingDefinition, |
| 424 | const DataLayout &dataLayout) { |
| 425 | return DeletionKind::Delete; |
| 426 | } |
| 427 | |
| 428 | LogicalResult LLVM::MemsetInlineOp::ensureOnlySafeAccesses( |
| 429 | const MemorySlot &slot, SmallVectorImpl<MemorySlot> &mustBeSafelyUsed, |
| 430 | const DataLayout &dataLayout) { |
| 431 | return success(IsSuccess: definitelyWritesOnlyWithinSlot(op: *this, slot, dataLayout)); |
| 432 | } |
| 433 | |
| 434 | bool LLVM::MemsetInlineOp::canRewire( |
| 435 | const DestructurableMemorySlot &slot, |
| 436 | SmallPtrSetImpl<Attribute> &usedIndices, |
| 437 | SmallVectorImpl<MemorySlot> &mustBeSafelyUsed, |
| 438 | const DataLayout &dataLayout) { |
| 439 | return memsetCanRewire(op: *this, slot, usedIndices, mustBeSafelyUsed, |
| 440 | dataLayout); |
| 441 | } |
| 442 | |
| 443 | DeletionKind |
| 444 | LLVM::MemsetInlineOp::rewire(const DestructurableMemorySlot &slot, |
| 445 | DenseMap<Attribute, MemorySlot> &subslots, |
| 446 | OpBuilder &builder, const DataLayout &dataLayout) { |
| 447 | return memsetRewire(op: *this, slot, subslots, builder, dataLayout); |
| 448 | } |
| 449 | |
| 450 | //===----------------------------------------------------------------------===// |
| 451 | // Interfaces for memcpy/memmove |
| 452 | //===----------------------------------------------------------------------===// |
| 453 | |
| 454 | template <class MemcpyLike> |
| 455 | static bool memcpyLoadsFrom(MemcpyLike op, const MemorySlot &slot) { |
| 456 | return op.getSrc() == slot.ptr; |
| 457 | } |
| 458 | |
| 459 | template <class MemcpyLike> |
| 460 | static bool memcpyStoresTo(MemcpyLike op, const MemorySlot &slot) { |
| 461 | return op.getDst() == slot.ptr; |
| 462 | } |
| 463 | |
| 464 | template <class MemcpyLike> |
| 465 | static Value memcpyGetStored(MemcpyLike op, const MemorySlot &slot, |
| 466 | OpBuilder &builder) { |
| 467 | return LLVM::LoadOp::create(builder, op.getLoc(), slot.valueType, |
| 468 | op.getSrc()); |
| 469 | } |
| 470 | |
| 471 | template <class MemcpyLike> |
| 472 | static bool |
| 473 | memcpyCanUsesBeRemoved(MemcpyLike op, const MemorySlot &slot, |
| 474 | const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| 475 | SmallVectorImpl<OpOperand *> &newBlockingUses, |
| 476 | const DataLayout &dataLayout) { |
| 477 | // If source and destination are the same, memcpy behavior is undefined and |
| 478 | // memmove is a no-op. Because there is no memory change happening here, |
| 479 | // simplifying such operations is left to canonicalization. |
| 480 | if (op.getDst() == op.getSrc()) |
| 481 | return false; |
| 482 | |
| 483 | if (op.getIsVolatile()) |
| 484 | return false; |
| 485 | |
| 486 | return getStaticMemIntrLen(op) == dataLayout.getTypeSize(t: slot.valueType); |
| 487 | } |
| 488 | |
| 489 | template <class MemcpyLike> |
| 490 | static DeletionKind |
| 491 | memcpyRemoveBlockingUses(MemcpyLike op, const MemorySlot &slot, |
| 492 | const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| 493 | OpBuilder &builder, Value reachingDefinition) { |
| 494 | if (op.loadsFrom(slot)) |
| 495 | LLVM::StoreOp::create(builder, op.getLoc(), reachingDefinition, |
| 496 | op.getDst()); |
| 497 | return DeletionKind::Delete; |
| 498 | } |
| 499 | |
| 500 | template <class MemcpyLike> |
| 501 | static LogicalResult |
| 502 | memcpyEnsureOnlySafeAccesses(MemcpyLike op, const MemorySlot &slot, |
| 503 | SmallVectorImpl<MemorySlot> &mustBeSafelyUsed) { |
| 504 | DataLayout dataLayout = DataLayout::closest(op); |
| 505 | // While rewiring memcpy-like intrinsics only supports full copies, partial |
| 506 | // copies are still safe accesses so it is enough to only check for writes |
| 507 | // within bounds. |
| 508 | return success(definitelyWritesOnlyWithinSlot(op, slot, dataLayout)); |
| 509 | } |
| 510 | |
| 511 | template <class MemcpyLike> |
| 512 | static bool memcpyCanRewire(MemcpyLike op, const DestructurableMemorySlot &slot, |
| 513 | SmallPtrSetImpl<Attribute> &usedIndices, |
| 514 | SmallVectorImpl<MemorySlot> &mustBeSafelyUsed, |
| 515 | const DataLayout &dataLayout) { |
| 516 | if (op.getIsVolatile()) |
| 517 | return false; |
| 518 | |
| 519 | if (!cast<DestructurableTypeInterface>(Val: slot.valueType) |
| 520 | .getSubelementIndexMap()) |
| 521 | return false; |
| 522 | |
| 523 | if (!areAllIndicesI32(slot)) |
| 524 | return false; |
| 525 | |
| 526 | // Only full copies are supported. |
| 527 | if (getStaticMemIntrLen(op) != dataLayout.getTypeSize(t: slot.valueType)) |
| 528 | return false; |
| 529 | |
| 530 | if (op.getSrc() == slot.ptr) |
| 531 | usedIndices.insert_range(R: llvm::make_first_range(c: slot.subelementTypes)); |
| 532 | |
| 533 | return true; |
| 534 | } |
| 535 | |
| 536 | namespace { |
| 537 | template <class MemcpyLike> |
| 538 | void createMemcpyLikeToReplace(OpBuilder &builder, const DataLayout &layout, |
| 539 | MemcpyLike toReplace, Value dst, Value src, |
| 540 | Type toCpy, bool isVolatile) { |
| 541 | Value memcpySize = |
| 542 | LLVM::ConstantOp::create(builder, toReplace.getLoc(), |
| 543 | IntegerAttr::get(toReplace.getLen().getType(), |
| 544 | layout.getTypeSize(t: toCpy))); |
| 545 | MemcpyLike::create(builder, toReplace.getLoc(), dst, src, memcpySize, |
| 546 | isVolatile); |
| 547 | } |
| 548 | |
| 549 | template <> |
| 550 | void createMemcpyLikeToReplace(OpBuilder &builder, const DataLayout &layout, |
| 551 | LLVM::MemcpyInlineOp toReplace, Value dst, |
| 552 | Value src, Type toCpy, bool isVolatile) { |
| 553 | Type lenType = IntegerType::get(context: toReplace->getContext(), |
| 554 | width: toReplace.getLen().getBitWidth()); |
| 555 | LLVM::MemcpyInlineOp::create( |
| 556 | builder, location: toReplace.getLoc(), dst, src, |
| 557 | len: IntegerAttr::get(type: lenType, value: layout.getTypeSize(t: toCpy)), isVolatile); |
| 558 | } |
| 559 | } // namespace |
| 560 | |
| 561 | /// Rewires a memcpy-like operation. Only copies to or from the full slot are |
| 562 | /// supported. |
| 563 | template <class MemcpyLike> |
| 564 | static DeletionKind |
| 565 | memcpyRewire(MemcpyLike op, const DestructurableMemorySlot &slot, |
| 566 | DenseMap<Attribute, MemorySlot> &subslots, OpBuilder &builder, |
| 567 | const DataLayout &dataLayout) { |
| 568 | if (subslots.empty()) |
| 569 | return DeletionKind::Delete; |
| 570 | |
| 571 | assert((slot.ptr == op.getDst()) != (slot.ptr == op.getSrc())); |
| 572 | bool isDst = slot.ptr == op.getDst(); |
| 573 | |
| 574 | #ifndef NDEBUG |
| 575 | size_t slotsTreated = 0; |
| 576 | #endif |
| 577 | |
| 578 | // It was previously checked that index types are consistent, so this type can |
| 579 | // be fetched now. |
| 580 | Type indexType = cast<IntegerAttr>(Val&: subslots.begin()->first).getType(); |
| 581 | for (size_t i = 0, e = slot.subelementTypes.size(); i != e; i++) { |
| 582 | Attribute index = IntegerAttr::get(type: indexType, value: i); |
| 583 | if (!subslots.contains(Val: index)) |
| 584 | continue; |
| 585 | const MemorySlot &subslot = subslots.at(Val: index); |
| 586 | |
| 587 | #ifndef NDEBUG |
| 588 | slotsTreated++; |
| 589 | #endif |
| 590 | |
| 591 | // First get a pointer to the equivalent of this subslot from the source |
| 592 | // pointer. |
| 593 | SmallVector<LLVM::GEPArg> gepIndices{ |
| 594 | 0, static_cast<int32_t>( |
| 595 | cast<IntegerAttr>(Val&: index).getValue().getZExtValue())}; |
| 596 | Value subslotPtrInOther = LLVM::GEPOp::create( |
| 597 | builder, op.getLoc(), LLVM::LLVMPointerType::get(context: op.getContext()), |
| 598 | slot.valueType, isDst ? op.getSrc() : op.getDst(), gepIndices); |
| 599 | |
| 600 | // Then create a new memcpy out of this source pointer. |
| 601 | createMemcpyLikeToReplace(builder, dataLayout, op, |
| 602 | isDst ? subslot.ptr : subslotPtrInOther, |
| 603 | isDst ? subslotPtrInOther : subslot.ptr, |
| 604 | subslot.valueType, op.getIsVolatile()); |
| 605 | } |
| 606 | |
| 607 | assert(subslots.size() == slotsTreated); |
| 608 | |
| 609 | return DeletionKind::Delete; |
| 610 | } |
| 611 | |
| 612 | bool LLVM::MemcpyOp::loadsFrom(const MemorySlot &slot) { |
| 613 | return memcpyLoadsFrom(op: *this, slot); |
| 614 | } |
| 615 | |
| 616 | bool LLVM::MemcpyOp::storesTo(const MemorySlot &slot) { |
| 617 | return memcpyStoresTo(op: *this, slot); |
| 618 | } |
| 619 | |
| 620 | Value LLVM::MemcpyOp::getStored(const MemorySlot &slot, OpBuilder &builder, |
| 621 | Value reachingDef, |
| 622 | const DataLayout &dataLayout) { |
| 623 | return memcpyGetStored(op: *this, slot, builder); |
| 624 | } |
| 625 | |
| 626 | bool LLVM::MemcpyOp::canUsesBeRemoved( |
| 627 | const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| 628 | SmallVectorImpl<OpOperand *> &newBlockingUses, |
| 629 | const DataLayout &dataLayout) { |
| 630 | return memcpyCanUsesBeRemoved(op: *this, slot, blockingUses, newBlockingUses, |
| 631 | dataLayout); |
| 632 | } |
| 633 | |
| 634 | DeletionKind LLVM::MemcpyOp::removeBlockingUses( |
| 635 | const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| 636 | OpBuilder &builder, Value reachingDefinition, |
| 637 | const DataLayout &dataLayout) { |
| 638 | return memcpyRemoveBlockingUses(op: *this, slot, blockingUses, builder, |
| 639 | reachingDefinition); |
| 640 | } |
| 641 | |
| 642 | LogicalResult LLVM::MemcpyOp::ensureOnlySafeAccesses( |
| 643 | const MemorySlot &slot, SmallVectorImpl<MemorySlot> &mustBeSafelyUsed, |
| 644 | const DataLayout &dataLayout) { |
| 645 | return memcpyEnsureOnlySafeAccesses(op: *this, slot, mustBeSafelyUsed); |
| 646 | } |
| 647 | |
| 648 | bool LLVM::MemcpyOp::canRewire(const DestructurableMemorySlot &slot, |
| 649 | SmallPtrSetImpl<Attribute> &usedIndices, |
| 650 | SmallVectorImpl<MemorySlot> &mustBeSafelyUsed, |
| 651 | const DataLayout &dataLayout) { |
| 652 | return memcpyCanRewire(op: *this, slot, usedIndices, mustBeSafelyUsed, |
| 653 | dataLayout); |
| 654 | } |
| 655 | |
| 656 | DeletionKind LLVM::MemcpyOp::rewire(const DestructurableMemorySlot &slot, |
| 657 | DenseMap<Attribute, MemorySlot> &subslots, |
| 658 | OpBuilder &builder, |
| 659 | const DataLayout &dataLayout) { |
| 660 | return memcpyRewire(op: *this, slot, subslots, builder, dataLayout); |
| 661 | } |
| 662 | |
| 663 | bool LLVM::MemcpyInlineOp::loadsFrom(const MemorySlot &slot) { |
| 664 | return memcpyLoadsFrom(op: *this, slot); |
| 665 | } |
| 666 | |
| 667 | bool LLVM::MemcpyInlineOp::storesTo(const MemorySlot &slot) { |
| 668 | return memcpyStoresTo(op: *this, slot); |
| 669 | } |
| 670 | |
| 671 | Value LLVM::MemcpyInlineOp::getStored(const MemorySlot &slot, |
| 672 | OpBuilder &builder, Value reachingDef, |
| 673 | const DataLayout &dataLayout) { |
| 674 | return memcpyGetStored(op: *this, slot, builder); |
| 675 | } |
| 676 | |
| 677 | bool LLVM::MemcpyInlineOp::canUsesBeRemoved( |
| 678 | const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| 679 | SmallVectorImpl<OpOperand *> &newBlockingUses, |
| 680 | const DataLayout &dataLayout) { |
| 681 | return memcpyCanUsesBeRemoved(op: *this, slot, blockingUses, newBlockingUses, |
| 682 | dataLayout); |
| 683 | } |
| 684 | |
| 685 | DeletionKind LLVM::MemcpyInlineOp::removeBlockingUses( |
| 686 | const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| 687 | OpBuilder &builder, Value reachingDefinition, |
| 688 | const DataLayout &dataLayout) { |
| 689 | return memcpyRemoveBlockingUses(op: *this, slot, blockingUses, builder, |
| 690 | reachingDefinition); |
| 691 | } |
| 692 | |
| 693 | LogicalResult LLVM::MemcpyInlineOp::ensureOnlySafeAccesses( |
| 694 | const MemorySlot &slot, SmallVectorImpl<MemorySlot> &mustBeSafelyUsed, |
| 695 | const DataLayout &dataLayout) { |
| 696 | return memcpyEnsureOnlySafeAccesses(op: *this, slot, mustBeSafelyUsed); |
| 697 | } |
| 698 | |
| 699 | bool LLVM::MemcpyInlineOp::canRewire( |
| 700 | const DestructurableMemorySlot &slot, |
| 701 | SmallPtrSetImpl<Attribute> &usedIndices, |
| 702 | SmallVectorImpl<MemorySlot> &mustBeSafelyUsed, |
| 703 | const DataLayout &dataLayout) { |
| 704 | return memcpyCanRewire(op: *this, slot, usedIndices, mustBeSafelyUsed, |
| 705 | dataLayout); |
| 706 | } |
| 707 | |
| 708 | DeletionKind |
| 709 | LLVM::MemcpyInlineOp::rewire(const DestructurableMemorySlot &slot, |
| 710 | DenseMap<Attribute, MemorySlot> &subslots, |
| 711 | OpBuilder &builder, const DataLayout &dataLayout) { |
| 712 | return memcpyRewire(op: *this, slot, subslots, builder, dataLayout); |
| 713 | } |
| 714 | |
| 715 | bool LLVM::MemmoveOp::loadsFrom(const MemorySlot &slot) { |
| 716 | return memcpyLoadsFrom(op: *this, slot); |
| 717 | } |
| 718 | |
| 719 | bool LLVM::MemmoveOp::storesTo(const MemorySlot &slot) { |
| 720 | return memcpyStoresTo(op: *this, slot); |
| 721 | } |
| 722 | |
| 723 | Value LLVM::MemmoveOp::getStored(const MemorySlot &slot, OpBuilder &builder, |
| 724 | Value reachingDef, |
| 725 | const DataLayout &dataLayout) { |
| 726 | return memcpyGetStored(op: *this, slot, builder); |
| 727 | } |
| 728 | |
| 729 | bool LLVM::MemmoveOp::canUsesBeRemoved( |
| 730 | const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| 731 | SmallVectorImpl<OpOperand *> &newBlockingUses, |
| 732 | const DataLayout &dataLayout) { |
| 733 | return memcpyCanUsesBeRemoved(op: *this, slot, blockingUses, newBlockingUses, |
| 734 | dataLayout); |
| 735 | } |
| 736 | |
| 737 | DeletionKind LLVM::MemmoveOp::removeBlockingUses( |
| 738 | const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| 739 | OpBuilder &builder, Value reachingDefinition, |
| 740 | const DataLayout &dataLayout) { |
| 741 | return memcpyRemoveBlockingUses(op: *this, slot, blockingUses, builder, |
| 742 | reachingDefinition); |
| 743 | } |
| 744 | |
| 745 | LogicalResult LLVM::MemmoveOp::ensureOnlySafeAccesses( |
| 746 | const MemorySlot &slot, SmallVectorImpl<MemorySlot> &mustBeSafelyUsed, |
| 747 | const DataLayout &dataLayout) { |
| 748 | return memcpyEnsureOnlySafeAccesses(op: *this, slot, mustBeSafelyUsed); |
| 749 | } |
| 750 | |
| 751 | bool LLVM::MemmoveOp::canRewire(const DestructurableMemorySlot &slot, |
| 752 | SmallPtrSetImpl<Attribute> &usedIndices, |
| 753 | SmallVectorImpl<MemorySlot> &mustBeSafelyUsed, |
| 754 | const DataLayout &dataLayout) { |
| 755 | return memcpyCanRewire(op: *this, slot, usedIndices, mustBeSafelyUsed, |
| 756 | dataLayout); |
| 757 | } |
| 758 | |
| 759 | DeletionKind LLVM::MemmoveOp::rewire(const DestructurableMemorySlot &slot, |
| 760 | DenseMap<Attribute, MemorySlot> &subslots, |
| 761 | OpBuilder &builder, |
| 762 | const DataLayout &dataLayout) { |
| 763 | return memcpyRewire(op: *this, slot, subslots, builder, dataLayout); |
| 764 | } |
| 765 | |
| 766 | SmallVector<Value> mlir::LLVM::MemcpyOp::getAccessedOperands() { |
| 767 | return {getDst(), getSrc()}; |
| 768 | } |
| 769 | |
| 770 | SmallVector<Value> mlir::LLVM::MemcpyInlineOp::getAccessedOperands() { |
| 771 | return {getDst(), getSrc()}; |
| 772 | } |
| 773 | |
| 774 | SmallVector<Value> mlir::LLVM::MemmoveOp::getAccessedOperands() { |
| 775 | return {getDst(), getSrc()}; |
| 776 | } |
| 777 | |
| 778 | SmallVector<Value> mlir::LLVM::MemsetOp::getAccessedOperands() { |
| 779 | return {getDst()}; |
| 780 | } |
| 781 | |
| 782 | SmallVector<Value> mlir::LLVM::MemsetInlineOp::getAccessedOperands() { |
| 783 | return {getDst()}; |
| 784 | } |
| 785 | |
| 786 | #define GET_OP_CLASSES |
| 787 | #include "mlir/Dialect/LLVMIR/LLVMMemoryIntrinsicOps.cpp.inc" |
| 788 | |