1//===- LLVMMemoryIntrinsicOps.cpp - LLVM dialect memory intrinsics --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the ops of op partition `MemoryIntrinsic` of the LLVM
10// dialect.
11//
12//===----------------------------------------------------------------------===//
13
14#include "mlir/Dialect/LLVMIR/LLVMMemoryIntrinsicOps.h"
15
16#include "IR/LLVMOps.h"
17#include "mlir/Dialect/LLVMIR/LLVMArithOps.h"
18#include "mlir/Dialect/LLVMIR/LLVMConstantOps.h"
19#include "mlir/Dialect/LLVMIR/LLVMConversionOps.h"
20#include "mlir/Dialect/LLVMIR/LLVMMemoryOps.h"
21#include "mlir/IR/Builders.h"
22#include "mlir/IR/DialectImplementation.h"
23#include "mlir/IR/Matchers.h"
24#include "mlir/Interfaces/DataLayoutInterfaces.h"
25#include "mlir/Interfaces/MemorySlotInterfaces.h"
26#include "llvm/ADT/STLExtras.h"
27#include "llvm/ADT/TypeSwitch.h"
28
29#include <optional>
30
31using namespace mlir;
32using namespace mlir::LLVM;
33
34bool LLVM::LifetimeStartOp::canUsesBeRemoved(
35 const SmallPtrSetImpl<OpOperand *> &blockingUses,
36 SmallVectorImpl<OpOperand *> &newBlockingUses,
37 const DataLayout &dataLayout) {
38 return true;
39}
40
41DeletionKind LLVM::LifetimeStartOp::removeBlockingUses(
42 const SmallPtrSetImpl<OpOperand *> &blockingUses, OpBuilder &builder) {
43 return DeletionKind::Delete;
44}
45
46bool LLVM::LifetimeEndOp::canUsesBeRemoved(
47 const SmallPtrSetImpl<OpOperand *> &blockingUses,
48 SmallVectorImpl<OpOperand *> &newBlockingUses,
49 const DataLayout &dataLayout) {
50 return true;
51}
52
53DeletionKind LLVM::LifetimeEndOp::removeBlockingUses(
54 const SmallPtrSetImpl<OpOperand *> &blockingUses, OpBuilder &builder) {
55 return DeletionKind::Delete;
56}
57
58bool LLVM::InvariantStartOp::canUsesBeRemoved(
59 const SmallPtrSetImpl<OpOperand *> &blockingUses,
60 SmallVectorImpl<OpOperand *> &newBlockingUses,
61 const DataLayout &dataLayout) {
62 return true;
63}
64
65DeletionKind LLVM::InvariantStartOp::removeBlockingUses(
66 const SmallPtrSetImpl<OpOperand *> &blockingUses, OpBuilder &builder) {
67 return DeletionKind::Delete;
68}
69
70bool LLVM::InvariantEndOp::canUsesBeRemoved(
71 const SmallPtrSetImpl<OpOperand *> &blockingUses,
72 SmallVectorImpl<OpOperand *> &newBlockingUses,
73 const DataLayout &dataLayout) {
74 return true;
75}
76
77DeletionKind LLVM::InvariantEndOp::removeBlockingUses(
78 const SmallPtrSetImpl<OpOperand *> &blockingUses, OpBuilder &builder) {
79 return DeletionKind::Delete;
80}
81
82bool LLVM::LaunderInvariantGroupOp::canUsesBeRemoved(
83 const SmallPtrSetImpl<OpOperand *> &blockingUses,
84 SmallVectorImpl<OpOperand *> &newBlockingUses,
85 const DataLayout &dataLayout) {
86 return forwardToUsers(op: *this, newBlockingUses);
87}
88
89DeletionKind LLVM::LaunderInvariantGroupOp::removeBlockingUses(
90 const SmallPtrSetImpl<OpOperand *> &blockingUses, OpBuilder &builder) {
91 return DeletionKind::Delete;
92}
93
94//===----------------------------------------------------------------------===//
95// Utilities for memory intrinsics
96//===----------------------------------------------------------------------===//
97
98namespace {
99/// Returns the length of the given memory intrinsic in bytes if it can be known
100/// at compile-time on a best-effort basis, nothing otherwise.
101template <class MemIntr>
102std::optional<uint64_t> getStaticMemIntrLen(MemIntr op) {
103 APInt memIntrLen;
104 if (!matchPattern(op.getLen(), m_ConstantInt(bind_value: &memIntrLen)))
105 return {};
106 if (memIntrLen.getBitWidth() > 64)
107 return {};
108 return memIntrLen.getZExtValue();
109}
110
111/// Returns the length of the given memory intrinsic in bytes if it can be known
112/// at compile-time on a best-effort basis, nothing otherwise.
113/// Because MemcpyInlineOp has its length encoded as an attribute, this requires
114/// specialized handling.
115template <>
116std::optional<uint64_t> getStaticMemIntrLen(LLVM::MemcpyInlineOp op) {
117 APInt memIntrLen = op.getLen();
118 if (memIntrLen.getBitWidth() > 64)
119 return {};
120 return memIntrLen.getZExtValue();
121}
122
123/// Returns the length of the given memory intrinsic in bytes if it can be known
124/// at compile-time on a best-effort basis, nothing otherwise.
125/// Because MemsetInlineOp has its length encoded as an attribute, this requires
126/// specialized handling.
127template <>
128std::optional<uint64_t> getStaticMemIntrLen(LLVM::MemsetInlineOp op) {
129 APInt memIntrLen = op.getLen();
130 if (memIntrLen.getBitWidth() > 64)
131 return {};
132 return memIntrLen.getZExtValue();
133}
134
135/// Returns an integer attribute representing the length of a memset intrinsic
136template <class MemsetIntr>
137IntegerAttr createMemsetLenAttr(MemsetIntr op) {
138 IntegerAttr memsetLenAttr;
139 bool successfulMatch =
140 matchPattern(op.getLen(), m_Constant<IntegerAttr>(bind_value: &memsetLenAttr));
141 (void)successfulMatch;
142 assert(successfulMatch);
143 return memsetLenAttr;
144}
145
146/// Returns an integer attribute representing the length of a memset intrinsic
147/// Because MemsetInlineOp has its length encoded as an attribute, this requires
148/// specialized handling.
149template <>
150IntegerAttr createMemsetLenAttr(LLVM::MemsetInlineOp op) {
151 return op.getLenAttr();
152}
153
154/// Creates a memset intrinsic of that matches the `toReplace` intrinsic
155/// using the provided parameters. There are template specializations for
156/// MemsetOp and MemsetInlineOp.
157template <class MemsetIntr>
158void createMemsetIntr(OpBuilder &builder, MemsetIntr toReplace,
159 IntegerAttr memsetLenAttr, uint64_t newMemsetSize,
160 DenseMap<Attribute, MemorySlot> &subslots,
161 Attribute index);
162
163template <>
164void createMemsetIntr(OpBuilder &builder, LLVM::MemsetOp toReplace,
165 IntegerAttr memsetLenAttr, uint64_t newMemsetSize,
166 DenseMap<Attribute, MemorySlot> &subslots,
167 Attribute index) {
168 Value newMemsetSizeValue =
169 LLVM::ConstantOp::create(
170 builder, location: toReplace.getLen().getLoc(),
171 value: IntegerAttr::get(type: memsetLenAttr.getType(), value: newMemsetSize))
172 .getResult();
173
174 LLVM::MemsetOp::create(builder, location: toReplace.getLoc(), dst: subslots.at(Val: index).ptr,
175 val: toReplace.getVal(), len: newMemsetSizeValue,
176 isVolatile: toReplace.getIsVolatile());
177}
178
179template <>
180void createMemsetIntr(OpBuilder &builder, LLVM::MemsetInlineOp toReplace,
181 IntegerAttr memsetLenAttr, uint64_t newMemsetSize,
182 DenseMap<Attribute, MemorySlot> &subslots,
183 Attribute index) {
184 auto newMemsetSizeValue =
185 IntegerAttr::get(type: memsetLenAttr.getType(), value: newMemsetSize);
186
187 LLVM::MemsetInlineOp::create(builder, location: toReplace.getLoc(),
188 dst: subslots.at(Val: index).ptr, val: toReplace.getVal(),
189 len: newMemsetSizeValue, isVolatile: toReplace.getIsVolatile());
190}
191} // namespace
192
193/// Returns whether one can be sure the memory intrinsic does not write outside
194/// of the bounds of the given slot, on a best-effort basis.
195template <class MemIntr>
196static bool definitelyWritesOnlyWithinSlot(MemIntr op, const MemorySlot &slot,
197 const DataLayout &dataLayout) {
198 if (!isa<LLVM::LLVMPointerType>(Val: slot.ptr.getType()) ||
199 op.getDst() != slot.ptr)
200 return false;
201
202 std::optional<uint64_t> memIntrLen = getStaticMemIntrLen(op);
203 return memIntrLen && *memIntrLen <= dataLayout.getTypeSize(t: slot.valueType);
204}
205
206/// Checks whether all indices are i32. This is used to check GEPs can index
207/// into them.
208static bool areAllIndicesI32(const DestructurableMemorySlot &slot) {
209 Type i32 = IntegerType::get(context: slot.ptr.getContext(), width: 32);
210 return llvm::all_of(Range: llvm::make_first_range(c: slot.subelementTypes),
211 P: [&](Attribute index) {
212 auto intIndex = dyn_cast<IntegerAttr>(Val&: index);
213 return intIndex && intIndex.getType() == i32;
214 });
215}
216
217//===----------------------------------------------------------------------===//
218// Interfaces for memset and memset.inline
219//===----------------------------------------------------------------------===//
220
221template <class MemsetIntr>
222static bool memsetCanRewire(MemsetIntr op, const DestructurableMemorySlot &slot,
223 SmallPtrSetImpl<Attribute> &usedIndices,
224 SmallVectorImpl<MemorySlot> &mustBeSafelyUsed,
225 const DataLayout &dataLayout) {
226 if (&slot.valueType.getDialect() != op.getOperation()->getDialect())
227 return false;
228
229 if (op.getIsVolatile())
230 return false;
231
232 if (!cast<DestructurableTypeInterface>(Val: slot.valueType)
233 .getSubelementIndexMap())
234 return false;
235
236 if (!areAllIndicesI32(slot))
237 return false;
238
239 return definitelyWritesOnlyWithinSlot(op, slot, dataLayout);
240}
241
242template <class MemsetIntr>
243static Value memsetGetStored(MemsetIntr op, const MemorySlot &slot,
244 OpBuilder &builder) {
245 /// Returns an integer value that is `width` bits wide representing the value
246 /// assigned to the slot by memset.
247 auto buildMemsetValue = [&](unsigned width) -> Value {
248 assert(width % 8 == 0);
249 auto intType = IntegerType::get(context: op.getContext(), width);
250
251 // If we know the pattern at compile time, we can compute and assign a
252 // constant directly.
253 IntegerAttr constantPattern;
254 if (matchPattern(op.getVal(), m_Constant(bind_value: &constantPattern))) {
255 assert(constantPattern.getValue().getBitWidth() == 8);
256 APInt memsetVal(/*numBits=*/width, /*val=*/0);
257 for (unsigned loBit = 0; loBit < width; loBit += 8)
258 memsetVal.insertBits(SubBits: constantPattern.getValue(), bitPosition: loBit);
259 return LLVM::ConstantOp::create(builder, op.getLoc(),
260 IntegerAttr::get(intType, memsetVal));
261 }
262
263 // If the output is a single byte, we can return the pattern directly.
264 if (width == 8)
265 return op.getVal();
266
267 // Otherwise build the memset integer at runtime by repeatedly shifting the
268 // value and or-ing it with the previous value.
269 uint64_t coveredBits = 8;
270 Value currentValue =
271 LLVM::ZExtOp::create(builder, op.getLoc(), intType, op.getVal());
272 while (coveredBits < width) {
273 Value shiftBy =
274 LLVM::ConstantOp::create(builder, op.getLoc(), intType, coveredBits);
275 Value shifted =
276 LLVM::ShlOp::create(builder, op.getLoc(), currentValue, shiftBy);
277 currentValue =
278 LLVM::OrOp::create(builder, op.getLoc(), currentValue, shifted);
279 coveredBits *= 2;
280 }
281
282 return currentValue;
283 };
284 return TypeSwitch<Type, Value>(slot.valueType)
285 .Case([&](IntegerType type) -> Value {
286 return buildMemsetValue(type.getWidth());
287 })
288 .Case([&](FloatType type) -> Value {
289 Value intVal = buildMemsetValue(type.getWidth());
290 return LLVM::BitcastOp::create(builder, op.getLoc(), type, intVal);
291 })
292 .DefaultUnreachable(
293 "getStored should not be called on memset to unsupported type");
294}
295
296template <class MemsetIntr>
297static bool
298memsetCanUsesBeRemoved(MemsetIntr op, const MemorySlot &slot,
299 const SmallPtrSetImpl<OpOperand *> &blockingUses,
300 SmallVectorImpl<OpOperand *> &newBlockingUses,
301 const DataLayout &dataLayout) {
302 bool canConvertType =
303 TypeSwitch<Type, bool>(slot.valueType)
304 .Case<IntegerType, FloatType>([](auto type) {
305 return type.getWidth() % 8 == 0 && type.getWidth() > 0;
306 })
307 .Default(false);
308 if (!canConvertType)
309 return false;
310
311 if (op.getIsVolatile())
312 return false;
313
314 return getStaticMemIntrLen(op) == dataLayout.getTypeSize(t: slot.valueType);
315}
316
317template <class MemsetIntr>
318static DeletionKind
319memsetRewire(MemsetIntr op, const DestructurableMemorySlot &slot,
320 DenseMap<Attribute, MemorySlot> &subslots, OpBuilder &builder,
321 const DataLayout &dataLayout) {
322
323 std::optional<DenseMap<Attribute, Type>> types =
324 cast<DestructurableTypeInterface>(Val: slot.valueType).getSubelementIndexMap();
325
326 IntegerAttr memsetLenAttr = createMemsetLenAttr(op);
327
328 auto structType = dyn_cast<LLVM::LLVMStructType>(Val: slot.valueType);
329
330 Type i32 = IntegerType::get(context: op.getContext(), width: 32);
331 uint64_t memsetLen = memsetLenAttr.getValue().getZExtValue();
332 for (size_t i = 0; i < types->size(); i++) {
333 // Create indices on the fly to get elements in the right order.
334 Attribute index = IntegerAttr::get(type: i32, value: i);
335 Type elemType = types->at(Val: index);
336 uint64_t typeSize = dataLayout.getTypeSize(t: elemType);
337 uint64_t offset = structType ? structType.getElementOffset(dataLayout, index: i)
338 : i * dataLayout.getTypeAllocSize(t: elemType);
339 if (offset >= memsetLen)
340 break;
341
342 // If this subslot is used, apply a new memset to it.
343 if (subslots.contains(Val: index)) {
344 uint64_t newMemsetSize = std::min(a: memsetLen - offset, b: typeSize);
345 createMemsetIntr(builder, op, memsetLenAttr, newMemsetSize, subslots,
346 index);
347 }
348 }
349
350 return DeletionKind::Delete;
351}
352
353bool LLVM::MemsetOp::loadsFrom(const MemorySlot &slot) { return false; }
354
355bool LLVM::MemsetOp::storesTo(const MemorySlot &slot) {
356 return getDst() == slot.ptr;
357}
358
359Value LLVM::MemsetOp::getStored(const MemorySlot &slot, OpBuilder &builder,
360 Value reachingDef,
361 const DataLayout &dataLayout) {
362 return memsetGetStored(op: *this, slot, builder);
363}
364
365bool LLVM::MemsetOp::canUsesBeRemoved(
366 const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
367 SmallVectorImpl<OpOperand *> &newBlockingUses,
368 const DataLayout &dataLayout) {
369 return memsetCanUsesBeRemoved(op: *this, slot, blockingUses, newBlockingUses,
370 dataLayout);
371}
372
373DeletionKind LLVM::MemsetOp::removeBlockingUses(
374 const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
375 OpBuilder &builder, Value reachingDefinition,
376 const DataLayout &dataLayout) {
377 return DeletionKind::Delete;
378}
379
380LogicalResult LLVM::MemsetOp::ensureOnlySafeAccesses(
381 const MemorySlot &slot, SmallVectorImpl<MemorySlot> &mustBeSafelyUsed,
382 const DataLayout &dataLayout) {
383 return success(IsSuccess: definitelyWritesOnlyWithinSlot(op: *this, slot, dataLayout));
384}
385
386bool LLVM::MemsetOp::canRewire(const DestructurableMemorySlot &slot,
387 SmallPtrSetImpl<Attribute> &usedIndices,
388 SmallVectorImpl<MemorySlot> &mustBeSafelyUsed,
389 const DataLayout &dataLayout) {
390 return memsetCanRewire(op: *this, slot, usedIndices, mustBeSafelyUsed,
391 dataLayout);
392}
393
394DeletionKind LLVM::MemsetOp::rewire(const DestructurableMemorySlot &slot,
395 DenseMap<Attribute, MemorySlot> &subslots,
396 OpBuilder &builder,
397 const DataLayout &dataLayout) {
398 return memsetRewire(op: *this, slot, subslots, builder, dataLayout);
399}
400
401bool LLVM::MemsetInlineOp::loadsFrom(const MemorySlot &slot) { return false; }
402
403bool LLVM::MemsetInlineOp::storesTo(const MemorySlot &slot) {
404 return getDst() == slot.ptr;
405}
406
407Value LLVM::MemsetInlineOp::getStored(const MemorySlot &slot,
408 OpBuilder &builder, Value reachingDef,
409 const DataLayout &dataLayout) {
410 return memsetGetStored(op: *this, slot, builder);
411}
412
413bool LLVM::MemsetInlineOp::canUsesBeRemoved(
414 const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
415 SmallVectorImpl<OpOperand *> &newBlockingUses,
416 const DataLayout &dataLayout) {
417 return memsetCanUsesBeRemoved(op: *this, slot, blockingUses, newBlockingUses,
418 dataLayout);
419}
420
421DeletionKind LLVM::MemsetInlineOp::removeBlockingUses(
422 const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
423 OpBuilder &builder, Value reachingDefinition,
424 const DataLayout &dataLayout) {
425 return DeletionKind::Delete;
426}
427
428LogicalResult LLVM::MemsetInlineOp::ensureOnlySafeAccesses(
429 const MemorySlot &slot, SmallVectorImpl<MemorySlot> &mustBeSafelyUsed,
430 const DataLayout &dataLayout) {
431 return success(IsSuccess: definitelyWritesOnlyWithinSlot(op: *this, slot, dataLayout));
432}
433
434bool LLVM::MemsetInlineOp::canRewire(
435 const DestructurableMemorySlot &slot,
436 SmallPtrSetImpl<Attribute> &usedIndices,
437 SmallVectorImpl<MemorySlot> &mustBeSafelyUsed,
438 const DataLayout &dataLayout) {
439 return memsetCanRewire(op: *this, slot, usedIndices, mustBeSafelyUsed,
440 dataLayout);
441}
442
443DeletionKind
444LLVM::MemsetInlineOp::rewire(const DestructurableMemorySlot &slot,
445 DenseMap<Attribute, MemorySlot> &subslots,
446 OpBuilder &builder, const DataLayout &dataLayout) {
447 return memsetRewire(op: *this, slot, subslots, builder, dataLayout);
448}
449
450//===----------------------------------------------------------------------===//
451// Interfaces for memcpy/memmove
452//===----------------------------------------------------------------------===//
453
454template <class MemcpyLike>
455static bool memcpyLoadsFrom(MemcpyLike op, const MemorySlot &slot) {
456 return op.getSrc() == slot.ptr;
457}
458
459template <class MemcpyLike>
460static bool memcpyStoresTo(MemcpyLike op, const MemorySlot &slot) {
461 return op.getDst() == slot.ptr;
462}
463
464template <class MemcpyLike>
465static Value memcpyGetStored(MemcpyLike op, const MemorySlot &slot,
466 OpBuilder &builder) {
467 return LLVM::LoadOp::create(builder, op.getLoc(), slot.valueType,
468 op.getSrc());
469}
470
471template <class MemcpyLike>
472static bool
473memcpyCanUsesBeRemoved(MemcpyLike op, const MemorySlot &slot,
474 const SmallPtrSetImpl<OpOperand *> &blockingUses,
475 SmallVectorImpl<OpOperand *> &newBlockingUses,
476 const DataLayout &dataLayout) {
477 // If source and destination are the same, memcpy behavior is undefined and
478 // memmove is a no-op. Because there is no memory change happening here,
479 // simplifying such operations is left to canonicalization.
480 if (op.getDst() == op.getSrc())
481 return false;
482
483 if (op.getIsVolatile())
484 return false;
485
486 return getStaticMemIntrLen(op) == dataLayout.getTypeSize(t: slot.valueType);
487}
488
489template <class MemcpyLike>
490static DeletionKind
491memcpyRemoveBlockingUses(MemcpyLike op, const MemorySlot &slot,
492 const SmallPtrSetImpl<OpOperand *> &blockingUses,
493 OpBuilder &builder, Value reachingDefinition) {
494 if (op.loadsFrom(slot))
495 LLVM::StoreOp::create(builder, op.getLoc(), reachingDefinition,
496 op.getDst());
497 return DeletionKind::Delete;
498}
499
500template <class MemcpyLike>
501static LogicalResult
502memcpyEnsureOnlySafeAccesses(MemcpyLike op, const MemorySlot &slot,
503 SmallVectorImpl<MemorySlot> &mustBeSafelyUsed) {
504 DataLayout dataLayout = DataLayout::closest(op);
505 // While rewiring memcpy-like intrinsics only supports full copies, partial
506 // copies are still safe accesses so it is enough to only check for writes
507 // within bounds.
508 return success(definitelyWritesOnlyWithinSlot(op, slot, dataLayout));
509}
510
511template <class MemcpyLike>
512static bool memcpyCanRewire(MemcpyLike op, const DestructurableMemorySlot &slot,
513 SmallPtrSetImpl<Attribute> &usedIndices,
514 SmallVectorImpl<MemorySlot> &mustBeSafelyUsed,
515 const DataLayout &dataLayout) {
516 if (op.getIsVolatile())
517 return false;
518
519 if (!cast<DestructurableTypeInterface>(Val: slot.valueType)
520 .getSubelementIndexMap())
521 return false;
522
523 if (!areAllIndicesI32(slot))
524 return false;
525
526 // Only full copies are supported.
527 if (getStaticMemIntrLen(op) != dataLayout.getTypeSize(t: slot.valueType))
528 return false;
529
530 if (op.getSrc() == slot.ptr)
531 usedIndices.insert_range(R: llvm::make_first_range(c: slot.subelementTypes));
532
533 return true;
534}
535
536namespace {
537template <class MemcpyLike>
538void createMemcpyLikeToReplace(OpBuilder &builder, const DataLayout &layout,
539 MemcpyLike toReplace, Value dst, Value src,
540 Type toCpy, bool isVolatile) {
541 Value memcpySize =
542 LLVM::ConstantOp::create(builder, toReplace.getLoc(),
543 IntegerAttr::get(toReplace.getLen().getType(),
544 layout.getTypeSize(t: toCpy)));
545 MemcpyLike::create(builder, toReplace.getLoc(), dst, src, memcpySize,
546 isVolatile);
547}
548
549template <>
550void createMemcpyLikeToReplace(OpBuilder &builder, const DataLayout &layout,
551 LLVM::MemcpyInlineOp toReplace, Value dst,
552 Value src, Type toCpy, bool isVolatile) {
553 Type lenType = IntegerType::get(context: toReplace->getContext(),
554 width: toReplace.getLen().getBitWidth());
555 LLVM::MemcpyInlineOp::create(
556 builder, location: toReplace.getLoc(), dst, src,
557 len: IntegerAttr::get(type: lenType, value: layout.getTypeSize(t: toCpy)), isVolatile);
558}
559} // namespace
560
561/// Rewires a memcpy-like operation. Only copies to or from the full slot are
562/// supported.
563template <class MemcpyLike>
564static DeletionKind
565memcpyRewire(MemcpyLike op, const DestructurableMemorySlot &slot,
566 DenseMap<Attribute, MemorySlot> &subslots, OpBuilder &builder,
567 const DataLayout &dataLayout) {
568 if (subslots.empty())
569 return DeletionKind::Delete;
570
571 assert((slot.ptr == op.getDst()) != (slot.ptr == op.getSrc()));
572 bool isDst = slot.ptr == op.getDst();
573
574#ifndef NDEBUG
575 size_t slotsTreated = 0;
576#endif
577
578 // It was previously checked that index types are consistent, so this type can
579 // be fetched now.
580 Type indexType = cast<IntegerAttr>(Val&: subslots.begin()->first).getType();
581 for (size_t i = 0, e = slot.subelementTypes.size(); i != e; i++) {
582 Attribute index = IntegerAttr::get(type: indexType, value: i);
583 if (!subslots.contains(Val: index))
584 continue;
585 const MemorySlot &subslot = subslots.at(Val: index);
586
587#ifndef NDEBUG
588 slotsTreated++;
589#endif
590
591 // First get a pointer to the equivalent of this subslot from the source
592 // pointer.
593 SmallVector<LLVM::GEPArg> gepIndices{
594 0, static_cast<int32_t>(
595 cast<IntegerAttr>(Val&: index).getValue().getZExtValue())};
596 Value subslotPtrInOther = LLVM::GEPOp::create(
597 builder, op.getLoc(), LLVM::LLVMPointerType::get(context: op.getContext()),
598 slot.valueType, isDst ? op.getSrc() : op.getDst(), gepIndices);
599
600 // Then create a new memcpy out of this source pointer.
601 createMemcpyLikeToReplace(builder, dataLayout, op,
602 isDst ? subslot.ptr : subslotPtrInOther,
603 isDst ? subslotPtrInOther : subslot.ptr,
604 subslot.valueType, op.getIsVolatile());
605 }
606
607 assert(subslots.size() == slotsTreated);
608
609 return DeletionKind::Delete;
610}
611
612bool LLVM::MemcpyOp::loadsFrom(const MemorySlot &slot) {
613 return memcpyLoadsFrom(op: *this, slot);
614}
615
616bool LLVM::MemcpyOp::storesTo(const MemorySlot &slot) {
617 return memcpyStoresTo(op: *this, slot);
618}
619
620Value LLVM::MemcpyOp::getStored(const MemorySlot &slot, OpBuilder &builder,
621 Value reachingDef,
622 const DataLayout &dataLayout) {
623 return memcpyGetStored(op: *this, slot, builder);
624}
625
626bool LLVM::MemcpyOp::canUsesBeRemoved(
627 const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
628 SmallVectorImpl<OpOperand *> &newBlockingUses,
629 const DataLayout &dataLayout) {
630 return memcpyCanUsesBeRemoved(op: *this, slot, blockingUses, newBlockingUses,
631 dataLayout);
632}
633
634DeletionKind LLVM::MemcpyOp::removeBlockingUses(
635 const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
636 OpBuilder &builder, Value reachingDefinition,
637 const DataLayout &dataLayout) {
638 return memcpyRemoveBlockingUses(op: *this, slot, blockingUses, builder,
639 reachingDefinition);
640}
641
642LogicalResult LLVM::MemcpyOp::ensureOnlySafeAccesses(
643 const MemorySlot &slot, SmallVectorImpl<MemorySlot> &mustBeSafelyUsed,
644 const DataLayout &dataLayout) {
645 return memcpyEnsureOnlySafeAccesses(op: *this, slot, mustBeSafelyUsed);
646}
647
648bool LLVM::MemcpyOp::canRewire(const DestructurableMemorySlot &slot,
649 SmallPtrSetImpl<Attribute> &usedIndices,
650 SmallVectorImpl<MemorySlot> &mustBeSafelyUsed,
651 const DataLayout &dataLayout) {
652 return memcpyCanRewire(op: *this, slot, usedIndices, mustBeSafelyUsed,
653 dataLayout);
654}
655
656DeletionKind LLVM::MemcpyOp::rewire(const DestructurableMemorySlot &slot,
657 DenseMap<Attribute, MemorySlot> &subslots,
658 OpBuilder &builder,
659 const DataLayout &dataLayout) {
660 return memcpyRewire(op: *this, slot, subslots, builder, dataLayout);
661}
662
663bool LLVM::MemcpyInlineOp::loadsFrom(const MemorySlot &slot) {
664 return memcpyLoadsFrom(op: *this, slot);
665}
666
667bool LLVM::MemcpyInlineOp::storesTo(const MemorySlot &slot) {
668 return memcpyStoresTo(op: *this, slot);
669}
670
671Value LLVM::MemcpyInlineOp::getStored(const MemorySlot &slot,
672 OpBuilder &builder, Value reachingDef,
673 const DataLayout &dataLayout) {
674 return memcpyGetStored(op: *this, slot, builder);
675}
676
677bool LLVM::MemcpyInlineOp::canUsesBeRemoved(
678 const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
679 SmallVectorImpl<OpOperand *> &newBlockingUses,
680 const DataLayout &dataLayout) {
681 return memcpyCanUsesBeRemoved(op: *this, slot, blockingUses, newBlockingUses,
682 dataLayout);
683}
684
685DeletionKind LLVM::MemcpyInlineOp::removeBlockingUses(
686 const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
687 OpBuilder &builder, Value reachingDefinition,
688 const DataLayout &dataLayout) {
689 return memcpyRemoveBlockingUses(op: *this, slot, blockingUses, builder,
690 reachingDefinition);
691}
692
693LogicalResult LLVM::MemcpyInlineOp::ensureOnlySafeAccesses(
694 const MemorySlot &slot, SmallVectorImpl<MemorySlot> &mustBeSafelyUsed,
695 const DataLayout &dataLayout) {
696 return memcpyEnsureOnlySafeAccesses(op: *this, slot, mustBeSafelyUsed);
697}
698
699bool LLVM::MemcpyInlineOp::canRewire(
700 const DestructurableMemorySlot &slot,
701 SmallPtrSetImpl<Attribute> &usedIndices,
702 SmallVectorImpl<MemorySlot> &mustBeSafelyUsed,
703 const DataLayout &dataLayout) {
704 return memcpyCanRewire(op: *this, slot, usedIndices, mustBeSafelyUsed,
705 dataLayout);
706}
707
708DeletionKind
709LLVM::MemcpyInlineOp::rewire(const DestructurableMemorySlot &slot,
710 DenseMap<Attribute, MemorySlot> &subslots,
711 OpBuilder &builder, const DataLayout &dataLayout) {
712 return memcpyRewire(op: *this, slot, subslots, builder, dataLayout);
713}
714
715bool LLVM::MemmoveOp::loadsFrom(const MemorySlot &slot) {
716 return memcpyLoadsFrom(op: *this, slot);
717}
718
719bool LLVM::MemmoveOp::storesTo(const MemorySlot &slot) {
720 return memcpyStoresTo(op: *this, slot);
721}
722
723Value LLVM::MemmoveOp::getStored(const MemorySlot &slot, OpBuilder &builder,
724 Value reachingDef,
725 const DataLayout &dataLayout) {
726 return memcpyGetStored(op: *this, slot, builder);
727}
728
729bool LLVM::MemmoveOp::canUsesBeRemoved(
730 const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
731 SmallVectorImpl<OpOperand *> &newBlockingUses,
732 const DataLayout &dataLayout) {
733 return memcpyCanUsesBeRemoved(op: *this, slot, blockingUses, newBlockingUses,
734 dataLayout);
735}
736
737DeletionKind LLVM::MemmoveOp::removeBlockingUses(
738 const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
739 OpBuilder &builder, Value reachingDefinition,
740 const DataLayout &dataLayout) {
741 return memcpyRemoveBlockingUses(op: *this, slot, blockingUses, builder,
742 reachingDefinition);
743}
744
745LogicalResult LLVM::MemmoveOp::ensureOnlySafeAccesses(
746 const MemorySlot &slot, SmallVectorImpl<MemorySlot> &mustBeSafelyUsed,
747 const DataLayout &dataLayout) {
748 return memcpyEnsureOnlySafeAccesses(op: *this, slot, mustBeSafelyUsed);
749}
750
751bool LLVM::MemmoveOp::canRewire(const DestructurableMemorySlot &slot,
752 SmallPtrSetImpl<Attribute> &usedIndices,
753 SmallVectorImpl<MemorySlot> &mustBeSafelyUsed,
754 const DataLayout &dataLayout) {
755 return memcpyCanRewire(op: *this, slot, usedIndices, mustBeSafelyUsed,
756 dataLayout);
757}
758
759DeletionKind LLVM::MemmoveOp::rewire(const DestructurableMemorySlot &slot,
760 DenseMap<Attribute, MemorySlot> &subslots,
761 OpBuilder &builder,
762 const DataLayout &dataLayout) {
763 return memcpyRewire(op: *this, slot, subslots, builder, dataLayout);
764}
765
766SmallVector<Value> mlir::LLVM::MemcpyOp::getAccessedOperands() {
767 return {getDst(), getSrc()};
768}
769
770SmallVector<Value> mlir::LLVM::MemcpyInlineOp::getAccessedOperands() {
771 return {getDst(), getSrc()};
772}
773
774SmallVector<Value> mlir::LLVM::MemmoveOp::getAccessedOperands() {
775 return {getDst(), getSrc()};
776}
777
778SmallVector<Value> mlir::LLVM::MemsetOp::getAccessedOperands() {
779 return {getDst()};
780}
781
782SmallVector<Value> mlir::LLVM::MemsetInlineOp::getAccessedOperands() {
783 return {getDst()};
784}
785
786#define GET_OP_CLASSES
787#include "mlir/Dialect/LLVMIR/LLVMMemoryIntrinsicOps.cpp.inc"
788