1//===- LLVMMemoryOps.cpp - LLVM dialect memory ops ------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the ops of op partition `Memory` of the LLVM dialect.
10//
11//===----------------------------------------------------------------------===//
12
13#include "mlir/Dialect/LLVMIR/LLVMMemoryOps.h"
14
15#include "IR/LLVMOps.h"
16#include "mlir/Dialect/LLVMIR/LLVMArithOps.h"
17#include "mlir/Dialect/LLVMIR/LLVMConstantOps.h"
18#include "mlir/Dialect/LLVMIR/LLVMConversionOps.h"
19#include "mlir/Dialect/LLVMIR/LLVMDebugIntrinsicOps.h"
20#include "mlir/IR/Builders.h"
21#include "mlir/IR/DialectImplementation.h"
22#include "mlir/IR/Matchers.h"
23#include "mlir/IR/PatternMatch.h"
24#include "mlir/Interfaces/DataLayoutInterfaces.h"
25#include "mlir/Interfaces/MemorySlotInterfaces.h"
26#include "llvm/ADT/STLExtras.h"
27#include "llvm/ADT/TypeSwitch.h"
28#include "llvm/Support/DebugLog.h"
29
30#include <optional>
31
32#define DEBUG_TYPE "sroa"
33
34using namespace mlir;
35using namespace mlir::LLVM;
36
37//===----------------------------------------------------------------------===//
38// Attribute Helpers
39//===----------------------------------------------------------------------===//
40
41static constexpr const char kElemTypeAttrName[] = "elem_type";
42
43//===----------------------------------------------------------------------===//
44// Printing, parsing, verification and canonicalization for LLVM::AllocaOp.
45//===----------------------------------------------------------------------===//
46
47void AllocaOp::print(OpAsmPrinter &p) {
48 auto funcTy =
49 FunctionType::get(context: getContext(), inputs: {getArraySize().getType()}, results: {getType()});
50
51 if (getInalloca())
52 p << " inalloca";
53
54 p << ' ' << getArraySize() << " x " << getElemType();
55 NamedAttrList attrs((*this)->getDiscardableAttrDictionary().getValue());
56 if (getAlignment() && *getAlignment() != 0)
57 attrs.append(name: getAlignmentAttrName(), attr: getAlignmentAttr());
58 p.printOptionalAttrDict(attrs);
59 p << " : " << funcTy;
60}
61
62// <operation> ::= `llvm.alloca` `inalloca`? ssa-use `x` type
63// attribute-dict? `:` type `,` type
64ParseResult AllocaOp::parse(OpAsmParser &parser, OperationState &result) {
65 OpAsmParser::UnresolvedOperand arraySize;
66 Type type, elemType;
67 SMLoc trailingTypeLoc;
68
69 if (succeeded(Result: parser.parseOptionalKeyword(keyword: "inalloca")))
70 result.addAttribute(name: getInallocaAttrName(name: result.name),
71 attr: UnitAttr::get(context: parser.getContext()));
72
73 if (parser.parseOperand(result&: arraySize) || parser.parseKeyword(keyword: "x") ||
74 parser.parseType(result&: elemType) ||
75 parser.parseOptionalAttrDict(result&: result.attributes) || parser.parseColon() ||
76 parser.getCurrentLocation(loc: &trailingTypeLoc) || parser.parseType(result&: type))
77 return failure();
78
79 std::optional<NamedAttribute> alignmentAttr =
80 result.attributes.getNamed(name: "alignment");
81 if (alignmentAttr.has_value()) {
82 auto alignmentInt = llvm::dyn_cast<IntegerAttr>(Val: alignmentAttr->getValue());
83 if (!alignmentInt)
84 return parser.emitError(loc: parser.getNameLoc(),
85 message: "expected integer alignment");
86 if (alignmentInt.getValue().isZero())
87 result.attributes.erase(name: "alignment");
88 }
89
90 // Extract the result type from the trailing function type.
91 auto funcType = llvm::dyn_cast<FunctionType>(Val&: type);
92 if (!funcType || funcType.getNumInputs() != 1 ||
93 funcType.getNumResults() != 1)
94 return parser.emitError(
95 loc: trailingTypeLoc,
96 message: "expected trailing function type with one argument and one result");
97
98 if (parser.resolveOperand(operand: arraySize, type: funcType.getInput(i: 0), result&: result.operands))
99 return failure();
100
101 Type resultType = funcType.getResult(i: 0);
102 if (auto ptrResultType = llvm::dyn_cast<LLVMPointerType>(Val&: resultType))
103 result.addAttribute(name: kElemTypeAttrName, attr: TypeAttr::get(type: elemType));
104
105 result.addTypes(newTypes: {funcType.getResult(i: 0)});
106 return success();
107}
108
109LogicalResult AllocaOp::verify() {
110 // Only certain target extension types can be used in 'alloca'.
111 if (auto targetExtType = dyn_cast<LLVMTargetExtType>(Val: getElemType());
112 targetExtType && !targetExtType.supportsMemOps())
113 return emitOpError()
114 << "this target extension type cannot be used in alloca";
115
116 return success();
117}
118
119LogicalResult AllocaOp::canonicalize(AllocaOp op, PatternRewriter &rewriter) {
120 // Convert `alloca Ty, C` to the canonical `alloca [C x Ty], 1` form.
121 APInt numElements;
122 if (!matchPattern(value: op.getArraySize(), pattern: m_ConstantInt(bind_value: &numElements)) ||
123 numElements.isOne() || numElements.getActiveBits() > 64)
124 return failure();
125
126 auto arrayType =
127 LLVMArrayType::get(elementType: op.getElemType(), numElements: numElements.getZExtValue());
128 Value one = ConstantOp::create(builder&: rewriter, location: op.getLoc(), type: rewriter.getI32Type(),
129 /*value=*/1);
130 auto newAlloca =
131 AllocaOp::create(builder&: rewriter, location: op.getLoc(), res: op.getType(), arraySize: one,
132 alignment: op.getAlignmentAttr(), elem_type: arrayType, inalloca: op.getInalloca());
133 newAlloca->setDiscardableAttrs(op->getDiscardableAttrDictionary());
134 rewriter.replaceOp(op, newOp: newAlloca);
135 return success();
136}
137
138//===----------------------------------------------------------------------===//
139// Code for LLVM::GEPOp.
140//===----------------------------------------------------------------------===//
141
142GEPIndicesAdaptor<ValueRange> GEPOp::getIndices() {
143 return GEPIndicesAdaptor<ValueRange>(getRawConstantIndicesAttr(),
144 getDynamicIndices());
145}
146
147/// Returns the elemental type of any LLVM-compatible vector type or self.
148Type mlir::LLVM::extractVectorElementType(Type type) {
149 if (auto vectorType = llvm::dyn_cast<VectorType>(Val&: type))
150 return vectorType.getElementType();
151 return type;
152}
153
154/// Destructures the 'indices' parameter into 'rawConstantIndices' and
155/// 'dynamicIndices', encoding the former in the process. In the process,
156/// dynamic indices which are used to index into a structure type are converted
157/// to constant indices when possible. To do this, the GEPs element type should
158/// be passed as first parameter.
159static void destructureIndices(Type currType, ArrayRef<GEPArg> indices,
160 SmallVectorImpl<int32_t> &rawConstantIndices,
161 SmallVectorImpl<Value> &dynamicIndices) {
162 for (const GEPArg &iter : indices) {
163 // If the thing we are currently indexing into is a struct we must turn
164 // any integer constants into constant indices. If this is not possible
165 // we don't do anything here. The verifier will catch it and emit a proper
166 // error. All other canonicalization is done in the fold method.
167 bool requiresConst = !rawConstantIndices.empty() &&
168 isa_and_nonnull<LLVMStructType>(Val: currType);
169 if (Value val = llvm::dyn_cast_if_present<Value>(Val: iter)) {
170 APInt intC;
171 if (requiresConst && matchPattern(value: val, pattern: m_ConstantInt(bind_value: &intC)) &&
172 intC.isSignedIntN(N: kGEPConstantBitWidth)) {
173 rawConstantIndices.push_back(Elt: intC.getSExtValue());
174 } else {
175 rawConstantIndices.push_back(Elt: GEPOp::kDynamicIndex);
176 dynamicIndices.push_back(Elt: val);
177 }
178 } else {
179 rawConstantIndices.push_back(Elt: cast<GEPConstantIndex>(Val: iter));
180 }
181
182 // Skip for very first iteration of this loop. First index does not index
183 // within the aggregates, but is just a pointer offset.
184 if (rawConstantIndices.size() == 1 || !currType)
185 continue;
186
187 currType = TypeSwitch<Type, Type>(currType)
188 .Case<VectorType, LLVMArrayType>(caseFn: [](auto containerType) {
189 return containerType.getElementType();
190 })
191 .Case(caseFn: [&](LLVMStructType structType) -> Type {
192 int64_t memberIndex = rawConstantIndices.back();
193 if (memberIndex >= 0 && static_cast<size_t>(memberIndex) <
194 structType.getBody().size())
195 return structType.getBody()[memberIndex];
196 return nullptr;
197 })
198 .Default(nullptr);
199 }
200}
201
202void GEPOp::build(OpBuilder &builder, OperationState &result, Type resultType,
203 Type elementType, Value basePtr, ArrayRef<GEPArg> indices,
204 GEPNoWrapFlags noWrapFlags, ConstantRangeAttr inrange,
205 ArrayRef<NamedAttribute> attributes) {
206 SmallVector<int32_t> rawConstantIndices;
207 SmallVector<Value> dynamicIndices;
208 destructureIndices(currType: elementType, indices, rawConstantIndices, dynamicIndices);
209
210 result.addTypes(newTypes: resultType);
211 result.addAttributes(newAttributes: attributes);
212 result.getOrAddProperties<Properties>().rawConstantIndices =
213 builder.getDenseI32ArrayAttr(values: rawConstantIndices);
214 result.getOrAddProperties<Properties>().noWrapFlags = noWrapFlags;
215 result.getOrAddProperties<Properties>().elem_type =
216 TypeAttr::get(type: elementType);
217 result.getOrAddProperties<Properties>().inrange = inrange;
218 result.addOperands(newOperands: basePtr);
219 result.addOperands(newOperands: dynamicIndices);
220}
221
222void GEPOp::build(OpBuilder &builder, OperationState &result, Type resultType,
223 Type elementType, Value basePtr, ValueRange indices,
224 GEPNoWrapFlags noWrapFlags, ConstantRangeAttr inrange,
225 ArrayRef<NamedAttribute> attributes) {
226 build(builder, result, resultType, elementType, basePtr,
227 indices: SmallVector<GEPArg>(indices), noWrapFlags, inrange, attributes);
228}
229
230ParseResult mlir::LLVM::parseGEPIndices(
231 OpAsmParser &parser,
232 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &indices,
233 DenseI32ArrayAttr &rawConstantIndices) {
234 SmallVector<int32_t> constantIndices;
235
236 auto idxParser = [&]() -> ParseResult {
237 int32_t constantIndex;
238 OptionalParseResult parsedInteger =
239 parser.parseOptionalInteger(result&: constantIndex);
240 if (parsedInteger.has_value()) {
241 if (failed(Result: parsedInteger.value()))
242 return failure();
243 constantIndices.push_back(Elt: constantIndex);
244 return success();
245 }
246
247 constantIndices.push_back(Elt: LLVM::GEPOp::kDynamicIndex);
248 return parser.parseOperand(result&: indices.emplace_back());
249 };
250 if (parser.parseCommaSeparatedList(parseElementFn: idxParser))
251 return failure();
252
253 rawConstantIndices =
254 DenseI32ArrayAttr::get(context: parser.getContext(), content: constantIndices);
255 return success();
256}
257
258void mlir::LLVM::printGEPIndices(OpAsmPrinter &printer, LLVM::GEPOp gepOp,
259 OperandRange indices,
260 DenseI32ArrayAttr rawConstantIndices) {
261 llvm::interleaveComma(
262 c: GEPIndicesAdaptor<OperandRange>(rawConstantIndices, indices), os&: printer,
263 each_fn: [&](PointerUnion<IntegerAttr, Value> cst) {
264 if (Value val = llvm::dyn_cast_if_present<Value>(Val&: cst))
265 printer.printOperand(value: val);
266 else
267 printer << cast<IntegerAttr>(Val&: cst).getInt();
268 });
269}
270
271/// For the given `indices`, check if they comply with `baseGEPType`,
272/// especially check against LLVMStructTypes nested within.
273static LogicalResult
274verifyStructIndices(Type baseGEPType, unsigned indexPos,
275 GEPIndicesAdaptor<ValueRange> indices,
276 function_ref<InFlightDiagnostic()> emitOpError) {
277 if (indexPos >= indices.size())
278 // Stop searching
279 return success();
280
281 return TypeSwitch<Type, LogicalResult>(baseGEPType)
282 .Case(caseFn: [&](LLVMStructType structType) -> LogicalResult {
283 auto attr = dyn_cast<IntegerAttr>(Val: indices[indexPos]);
284 if (!attr)
285 return emitOpError() << "expected index " << indexPos
286 << " indexing a struct to be constant";
287
288 int32_t gepIndex = attr.getInt();
289 ArrayRef<Type> elementTypes = structType.getBody();
290 if (gepIndex < 0 ||
291 static_cast<size_t>(gepIndex) >= elementTypes.size())
292 return emitOpError() << "index " << indexPos
293 << " indexing a struct is out of bounds";
294
295 // Instead of recursively going into every children types, we only
296 // dive into the one indexed by gepIndex.
297 return verifyStructIndices(baseGEPType: elementTypes[gepIndex], indexPos: indexPos + 1,
298 indices, emitOpError);
299 })
300 .Case<VectorType, LLVMArrayType>(
301 caseFn: [&](auto containerType) -> LogicalResult {
302 return verifyStructIndices(containerType.getElementType(),
303 indexPos + 1, indices, emitOpError);
304 })
305 .Default(defaultFn: [&](auto otherType) -> LogicalResult {
306 return emitOpError()
307 << "type " << otherType << " cannot be indexed (index #"
308 << indexPos << ")";
309 });
310}
311
312/// Driver function around `verifyStructIndices`.
313static LogicalResult
314verifyStructIndices(Type baseGEPType, GEPIndicesAdaptor<ValueRange> indices,
315 function_ref<InFlightDiagnostic()> emitOpError) {
316 return verifyStructIndices(baseGEPType, /*indexPos=*/1, indices, emitOpError);
317}
318
319LogicalResult LLVM::GEPOp::verify() {
320 if (static_cast<size_t>(
321 llvm::count(Range: getRawConstantIndices(), Element: kDynamicIndex)) !=
322 getDynamicIndices().size())
323 return emitOpError(message: "expected as many dynamic indices as specified in '")
324 << getRawConstantIndicesAttrName().getValue() << "'";
325
326 if (getNoWrapFlags() == GEPNoWrapFlags::inboundsFlag)
327 return emitOpError(message: "'inbounds_flag' cannot be used directly.");
328
329 // LLVM stores `inrange` only on GetElementPtrConstantExpr, whose operands
330 // are already Constants. This dialect has one GEP operation, so proving the
331 // base and indices are constant expressions means walking SSA. A local check
332 // rejects valid cases such as a nested constant GEP base or a `ptrtoint`
333 // index.
334 if (auto inrange = getInrangeAttr()) {
335 auto pointerType =
336 cast<LLVMPointerType>(Val: extractVectorElementType(type: getBase().getType()));
337 DataLayout dataLayout = DataLayout::closest(op: *this);
338 std::optional<uint64_t> indexWidth =
339 dataLayout.getTypeIndexBitwidth(t: pointerType);
340 assert(indexWidth && "pointers always return an index bitwidth");
341 if (inrange.getLower().getBitWidth() != *indexWidth)
342 return emitOpError(message: "'inrange' bitwidth ")
343 << inrange.getLower().getBitWidth()
344 << " must match the pointer index bitwidth (" << *indexWidth
345 << ") specified in the datalayout";
346 if (inrange.getLower().sge(RHS: inrange.getUpper()))
347 return emitOpError(message: "expected 'inrange' end to be larger than start");
348 }
349
350 return verifyStructIndices(baseGEPType: getElemType(), indices: getIndices(),
351 emitOpError: [&] { return emitOpError(); });
352}
353
354//===----------------------------------------------------------------------===//
355// LoadOp
356//===----------------------------------------------------------------------===//
357
358void LoadOp::getEffects(
359 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
360 &effects) {
361 effects.emplace_back(Args: MemoryEffects::Read::get(), Args: &getAddrMutable());
362 // Volatile operations can have target-specific read-write effects on
363 // memory besides the one referred to by the pointer operand.
364 // Similarly, atomic operations that are monotonic or stricter cause
365 // synchronization that from a language point-of-view, are arbitrary
366 // read-writes into memory.
367 if (getVolatile_() || (getOrdering() != AtomicOrdering::not_atomic &&
368 getOrdering() != AtomicOrdering::unordered)) {
369 effects.emplace_back(Args: MemoryEffects::Write::get());
370 effects.emplace_back(Args: MemoryEffects::Read::get());
371 }
372}
373
374/// Returns true if the given type is supported by atomic operations. All
375/// integer, float, and pointer types with a power-of-two bitsize and a minimal
376/// size of 8 bits are supported.
377bool LLVM::isTypeCompatibleWithAtomicOp(Type type,
378 const DataLayout &dataLayout) {
379 if (!isa<IntegerType, LLVMPointerType>(Val: type))
380 if (!isCompatibleFloatingPointType(type))
381 return false;
382
383 llvm::TypeSize bitWidth = dataLayout.getTypeSizeInBits(t: type);
384 if (bitWidth.isScalable())
385 return false;
386 // Needs to be at least 8 bits and a power of two.
387 return bitWidth >= 8 && (bitWidth & (bitWidth - 1)) == 0;
388}
389
390/// Verifies the attributes and the type of atomic memory access operations.
391template <typename OpTy>
392static LogicalResult
393verifyAtomicMemOp(OpTy memOp, Type valueType,
394 ArrayRef<AtomicOrdering> unsupportedOrderings) {
395 if (memOp.getOrdering() != AtomicOrdering::not_atomic) {
396 DataLayout dataLayout = DataLayout::closest(op: memOp);
397 if (!isTypeCompatibleWithAtomicOp(type: valueType, dataLayout))
398 return memOp.emitOpError("unsupported type ")
399 << valueType << " for atomic access";
400 if (llvm::is_contained(unsupportedOrderings, memOp.getOrdering()))
401 return memOp.emitOpError("unsupported ordering '")
402 << stringifyAtomicOrdering(memOp.getOrdering()) << "'";
403 if (!memOp.getAlignment())
404 return memOp.emitOpError("expected alignment for atomic access");
405 return success();
406 }
407 if (memOp.getSyncscope())
408 return memOp.emitOpError(
409 "expected syncscope to be null for non-atomic access");
410 return success();
411}
412
413LogicalResult LoadOp::verify() {
414 Type valueType = getResult().getType();
415 return verifyAtomicMemOp(memOp: *this, valueType,
416 unsupportedOrderings: {AtomicOrdering::release, AtomicOrdering::acq_rel});
417}
418
419void LoadOp::build(OpBuilder &builder, OperationState &state, Type type,
420 Value addr, unsigned alignment, bool isVolatile,
421 bool isNonTemporal, bool isInvariant, bool isInvariantGroup,
422 bool isNoundef, AtomicOrdering ordering,
423 StringRef syncscope) {
424 build(odsBuilder&: builder, odsState&: state, res: type, addr,
425 alignment: alignment ? builder.getI64IntegerAttr(value: alignment) : nullptr, volatile_: isVolatile,
426 nontemporal: isNonTemporal, invariant: isInvariant, invariantGroup: isInvariantGroup, ordering,
427 syncscope: syncscope.empty() ? nullptr : builder.getStringAttr(bytes: syncscope),
428 /*dereferenceable=*/nullptr, noundef: isNoundef, /*range=*/nullptr,
429 /*access_groups=*/nullptr,
430 /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr,
431 /*tbaa=*/nullptr);
432}
433
434//===----------------------------------------------------------------------===//
435// StoreOp
436//===----------------------------------------------------------------------===//
437
438void StoreOp::getEffects(
439 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
440 &effects) {
441 effects.emplace_back(Args: MemoryEffects::Write::get(), Args: &getAddrMutable());
442 // Volatile operations can have target-specific read-write effects on
443 // memory besides the one referred to by the pointer operand.
444 // Similarly, atomic operations that are monotonic or stricter cause
445 // synchronization that from a language point-of-view, are arbitrary
446 // read-writes into memory.
447 if (getVolatile_() || (getOrdering() != AtomicOrdering::not_atomic &&
448 getOrdering() != AtomicOrdering::unordered)) {
449 effects.emplace_back(Args: MemoryEffects::Write::get());
450 effects.emplace_back(Args: MemoryEffects::Read::get());
451 }
452}
453
454LogicalResult StoreOp::verify() {
455 Type valueType = getValue().getType();
456 return verifyAtomicMemOp(memOp: *this, valueType,
457 unsupportedOrderings: {AtomicOrdering::acquire, AtomicOrdering::acq_rel});
458}
459
460void StoreOp::build(OpBuilder &builder, OperationState &state, Value value,
461 Value addr, unsigned alignment, bool isVolatile,
462 bool isNonTemporal, bool isInvariantGroup,
463 AtomicOrdering ordering, StringRef syncscope) {
464 build(odsBuilder&: builder, odsState&: state, value, addr,
465 alignment: alignment ? builder.getI64IntegerAttr(value: alignment) : nullptr, volatile_: isVolatile,
466 nontemporal: isNonTemporal, invariantGroup: isInvariantGroup, ordering,
467 syncscope: syncscope.empty() ? nullptr : builder.getStringAttr(bytes: syncscope),
468 /*access_groups=*/nullptr,
469 /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
470}
471
472//===----------------------------------------------------------------------===//
473// AtomicRMWOp
474//===----------------------------------------------------------------------===//
475
476void AtomicRMWOp::build(OpBuilder &builder, OperationState &state,
477 AtomicBinOp binOp, Value ptr, Value val,
478 AtomicOrdering ordering, StringRef syncscope,
479 unsigned alignment, bool isVolatile,
480 bool ignoreDenormalMode) {
481 build(odsBuilder&: builder, odsState&: state, res: val.getType(), bin_op: binOp, ptr, val, ordering,
482 syncscope: !syncscope.empty() ? builder.getStringAttr(bytes: syncscope) : nullptr,
483 alignment: alignment ? builder.getI64IntegerAttr(value: alignment) : nullptr, volatile_: isVolatile,
484 ignore_denormal_mode: ignoreDenormalMode,
485 /*access_groups=*/nullptr,
486 /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
487}
488
489LogicalResult AtomicRMWOp::verify() {
490 auto valType = getVal().getType();
491 if (isFloatingPointOperation()) {
492 if (isCompatibleVectorType(type: valType)) {
493 if (isScalableVectorType(vectorType: valType))
494 return emitOpError(message: "expected LLVM IR fixed vector type");
495 Type elemType = llvm::cast<VectorType>(Val&: valType).getElementType();
496 if (!isCompatibleFloatingPointType(type: elemType))
497 return emitOpError(
498 message: "expected LLVM IR floating point type for vector element");
499 } else if (!isCompatibleFloatingPointType(type: valType)) {
500 return emitOpError(message: "expected LLVM IR floating point type");
501 }
502 } else if (getBinOp() == AtomicBinOp::xchg) {
503 DataLayout dataLayout = DataLayout::closest(op: *this);
504 if (!isTypeCompatibleWithAtomicOp(type: valType, dataLayout))
505 return emitOpError(message: "unexpected LLVM IR type for 'xchg' bin_op");
506 } else {
507 auto intType = llvm::dyn_cast<IntegerType>(Val&: valType);
508 unsigned intBitWidth = intType ? intType.getWidth() : 0;
509 if (intBitWidth != 8 && intBitWidth != 16 && intBitWidth != 32 &&
510 intBitWidth != 64)
511 return emitOpError(message: "expected LLVM IR integer type");
512 }
513
514 if (getIgnoreDenormalMode() && !isFloatingPointOperation())
515 return emitOpError(
516 message: "expected floating-point operation with 'ignore_denormal_mode'");
517
518 if (static_cast<unsigned>(getOrdering()) <
519 static_cast<unsigned>(AtomicOrdering::monotonic))
520 return emitOpError() << "expected at least '"
521 << stringifyAtomicOrdering(AtomicOrdering::monotonic)
522 << "' ordering";
523
524 return success();
525}
526
527//===----------------------------------------------------------------------===//
528// AtomicCmpXchgOp
529//===----------------------------------------------------------------------===//
530
531/// Returns an LLVM struct type that contains a value type and a boolean type.
532LLVMStructType mlir::LLVM::getValAndBoolStructType(Type valType) {
533 auto boolType = IntegerType::get(context: valType.getContext(), width: 1);
534 return LLVMStructType::getLiteral(context: valType.getContext(), types: {valType, boolType});
535}
536
537void AtomicCmpXchgOp::build(OpBuilder &builder, OperationState &state,
538 Value ptr, Value cmp, Value val,
539 AtomicOrdering successOrdering,
540 AtomicOrdering failureOrdering, StringRef syncscope,
541 unsigned alignment, bool isWeak, bool isVolatile) {
542 build(odsBuilder&: builder, odsState&: state, res: getValAndBoolStructType(valType: val.getType()), ptr, cmp, val,
543 success_ordering: successOrdering, failure_ordering: failureOrdering,
544 syncscope: !syncscope.empty() ? builder.getStringAttr(bytes: syncscope) : nullptr,
545 alignment: alignment ? builder.getI64IntegerAttr(value: alignment) : nullptr, weak: isWeak,
546 volatile_: isVolatile, /*access_groups=*/nullptr,
547 /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
548}
549
550LogicalResult AtomicCmpXchgOp::verify() {
551 auto ptrType = llvm::cast<LLVM::LLVMPointerType>(Val: getPtr().getType());
552 if (!ptrType)
553 return emitOpError(message: "expected LLVM IR pointer type for operand #0");
554 auto valType = getVal().getType();
555 DataLayout dataLayout = DataLayout::closest(op: *this);
556 if (!isTypeCompatibleWithAtomicOp(type: valType, dataLayout))
557 return emitOpError(message: "unexpected LLVM IR type");
558 if (getSuccessOrdering() < AtomicOrdering::monotonic ||
559 getFailureOrdering() < AtomicOrdering::monotonic)
560 return emitOpError(message: "ordering must be at least 'monotonic'");
561 if (getFailureOrdering() == AtomicOrdering::release ||
562 getFailureOrdering() == AtomicOrdering::acq_rel)
563 return emitOpError(message: "failure ordering cannot be 'release' or 'acq_rel'");
564 return success();
565}
566
567//===----------------------------------------------------------------------===//
568// FenceOp
569//===----------------------------------------------------------------------===//
570
571void FenceOp::build(OpBuilder &builder, OperationState &state,
572 AtomicOrdering ordering, StringRef syncscope) {
573 build(odsBuilder&: builder, odsState&: state, ordering,
574 syncscope: syncscope.empty() ? nullptr : builder.getStringAttr(bytes: syncscope));
575}
576
577LogicalResult FenceOp::verify() {
578 if (getOrdering() == AtomicOrdering::not_atomic ||
579 getOrdering() == AtomicOrdering::unordered ||
580 getOrdering() == AtomicOrdering::monotonic)
581 return emitOpError(message: "can be given only acquire, release, acq_rel, "
582 "and seq_cst orderings");
583 return success();
584}
585
586//===----------------------------------------------------------------------===//
587// Folder for LLVM::GEPOp
588//===----------------------------------------------------------------------===//
589
590OpFoldResult LLVM::GEPOp::fold(FoldAdaptor adaptor) {
591 GEPIndicesAdaptor<ArrayRef<Attribute>> indices(getRawConstantIndicesAttr(),
592 adaptor.getDynamicIndices());
593
594 // Avoid losing inrange information.
595 if (getInrangeAttr())
596 return {};
597
598 // gep %x:T, 0 -> %x
599 if (getBase().getType() == getType() && indices.size() == 1)
600 if (auto integer = llvm::dyn_cast_or_null<IntegerAttr>(Val: indices[0]))
601 if (integer.getValue().isZero())
602 return getBase();
603
604 // Canonicalize any dynamic indices of constant value to constant indices.
605 bool changed = false;
606 SmallVector<GEPArg> gepArgs;
607 for (auto iter : llvm::enumerate(First&: indices)) {
608 auto integer = llvm::dyn_cast_or_null<IntegerAttr>(Val&: iter.value());
609 // Constant indices can only be int32_t, so if integer does not fit we
610 // are forced to keep it dynamic, despite being a constant.
611 if (!indices.isDynamicIndex(index: iter.index()) || !integer ||
612 !integer.getValue().isSignedIntN(N: kGEPConstantBitWidth)) {
613
614 PointerUnion<IntegerAttr, Value> existing = getIndices()[iter.index()];
615 if (Value val = llvm::dyn_cast_if_present<Value>(Val&: existing))
616 gepArgs.emplace_back(Args&: val);
617 else
618 gepArgs.emplace_back(Args: cast<IntegerAttr>(Val&: existing).getInt());
619
620 continue;
621 }
622
623 changed = true;
624 gepArgs.emplace_back(Args: integer.getInt());
625 }
626 if (changed) {
627 SmallVector<int32_t> rawConstantIndices;
628 SmallVector<Value> dynamicIndices;
629 destructureIndices(currType: getElemType(), indices: gepArgs, rawConstantIndices,
630 dynamicIndices);
631
632 getDynamicIndicesMutable().assign(values: dynamicIndices);
633 setRawConstantIndices(rawConstantIndices);
634 return Value{*this};
635 }
636
637 return {};
638}
639
640Value LLVM::GEPOp::getViewSource() { return getBase(); }
641
642//===----------------------------------------------------------------------===//
643// Interfaces for AllocaOp
644//===----------------------------------------------------------------------===//
645
646llvm::SmallVector<MemorySlot> LLVM::AllocaOp::getPromotableSlots() {
647 if (!getOperation()->getBlock()->isEntryBlock())
648 return {};
649
650 return {MemorySlot{.ptr: getResult(), .valueType: getElemType()}};
651}
652
653Value LLVM::AllocaOp::getDefaultValue(const MemorySlot &slot,
654 OpBuilder &builder) {
655 return LLVM::UndefOp::create(builder, location: getLoc(), res: slot.valueType);
656}
657
658void LLVM::AllocaOp::handleBlockArgument(const MemorySlot &slot,
659 BlockArgument argument,
660 OpBuilder &builder) {
661 for (Operation *user : getOperation()->getUsers())
662 if (auto declareOp = llvm::dyn_cast<LLVM::DbgDeclareOp>(Val: user))
663 LLVM::DbgValueOp::create(builder, location: declareOp.getLoc(), value: argument,
664 varInfo: declareOp.getVarInfo(),
665 locationExpr: declareOp.getLocationExpr());
666}
667
668std::optional<PromotableAllocationOpInterface>
669LLVM::AllocaOp::handlePromotionComplete(const MemorySlot &slot,
670 Value defaultValue,
671 OpBuilder &builder) {
672 if (defaultValue && defaultValue.use_empty())
673 defaultValue.getDefiningOp()->erase();
674 this->erase();
675 return std::nullopt;
676}
677
678SmallVector<DestructurableMemorySlot> LLVM::AllocaOp::getDestructurableSlots() {
679 if (!mlir::matchPattern(value: getArraySize(), pattern: m_One()))
680 return {};
681
682 auto destructurable = dyn_cast<DestructurableTypeInterface>(Val: getElemType());
683 if (!destructurable)
684 return {};
685
686 std::optional<DenseMap<Attribute, Type>> destructuredType =
687 destructurable.getSubelementIndexMap();
688 if (!destructuredType)
689 return {};
690
691 return {DestructurableMemorySlot{{.ptr: getResult(), .valueType: getElemType()},
692 .subelementTypes: *destructuredType}};
693}
694
695DenseMap<Attribute, MemorySlot> LLVM::AllocaOp::destructure(
696 const DestructurableMemorySlot &slot,
697 const SmallPtrSetImpl<Attribute> &usedIndices, OpBuilder &builder,
698 SmallVectorImpl<DestructurableAllocationOpInterface> &newAllocators) {
699 assert(slot.ptr == getResult());
700 builder.setInsertionPointAfter(*this);
701
702 auto destructurableType = cast<DestructurableTypeInterface>(Val: getElemType());
703 DenseMap<Attribute, MemorySlot> slotMap;
704 // Iterate subelements in their original type order to produce allocas in a
705 // deterministic, readable order (matching appearance in the source type).
706 Type i32 = IntegerType::get(context: getContext(), width: 32);
707 for (size_t i = 0; i < slot.subelementTypes.size(); i++) {
708 Attribute index = IntegerAttr::get(type: i32, value: i);
709 if (!usedIndices.contains(Ptr: index))
710 continue;
711 Type elemType = destructurableType.getTypeAtIndex(index);
712 assert(elemType && "used index must exist");
713 auto subAlloca = LLVM::AllocaOp::create(
714 builder, location: getLoc(), resultType: LLVM::LLVMPointerType::get(context: getContext()), elementType: elemType,
715 arraySize: getArraySize());
716 newAllocators.push_back(Elt: subAlloca);
717 slotMap.try_emplace<MemorySlot>(Key: index, Args: {.ptr: subAlloca.getResult(), .valueType: elemType});
718 }
719
720 return slotMap;
721}
722
723std::optional<DestructurableAllocationOpInterface>
724LLVM::AllocaOp::handleDestructuringComplete(
725 const DestructurableMemorySlot &slot, OpBuilder &builder) {
726 assert(slot.ptr == getResult());
727 this->erase();
728 return std::nullopt;
729}
730
731//===----------------------------------------------------------------------===//
732// Interfaces for LoadOp/StoreOp
733//===----------------------------------------------------------------------===//
734
735bool LLVM::LoadOp::loadsFrom(const MemorySlot &slot) {
736 return getAddr() == slot.ptr;
737}
738
739bool LLVM::LoadOp::storesTo(const MemorySlot &slot) { return false; }
740
741Value LLVM::LoadOp::getStored(const MemorySlot &slot, OpBuilder &builder,
742 Value reachingDef, const DataLayout &dataLayout) {
743 llvm_unreachable("getStored should not be called on LoadOp");
744}
745
746bool LLVM::StoreOp::loadsFrom(const MemorySlot &slot) { return false; }
747
748bool LLVM::StoreOp::storesTo(const MemorySlot &slot) {
749 return getAddr() == slot.ptr;
750}
751
752/// Checks if `type` can be used in any kind of conversion sequences.
753static bool isSupportedTypeForConversion(Type type) {
754 // Aggregate types are not bitcastable.
755 if (isa<LLVM::LLVMStructType, LLVM::LLVMArrayType>(Val: type))
756 return false;
757
758 if (auto vectorType = dyn_cast<VectorType>(Val&: type)) {
759 // Vectors of pointers cannot be casted.
760 if (isa<LLVM::LLVMPointerType>(Val: vectorType.getElementType()))
761 return false;
762 // Scalable types are not supported.
763 return !vectorType.isScalable();
764 }
765 return true;
766}
767
768/// Checks that `rhs` can be converted to `lhs` by a sequence of casts and
769/// truncations. Checks for narrowing or widening conversion compatibility
770/// depending on `narrowingConversion`.
771static bool areConversionCompatible(const DataLayout &layout, Type targetType,
772 Type srcType, bool narrowingConversion) {
773 if (targetType == srcType)
774 return true;
775
776 if (!isSupportedTypeForConversion(type: targetType) ||
777 !isSupportedTypeForConversion(type: srcType))
778 return false;
779
780 uint64_t targetSize = layout.getTypeSize(t: targetType);
781 uint64_t srcSize = layout.getTypeSize(t: srcType);
782
783 // Pointer casts will only be sane when the bitsize of both pointer types is
784 // the same.
785 if (isa<LLVM::LLVMPointerType>(Val: targetType) &&
786 isa<LLVM::LLVMPointerType>(Val: srcType))
787 return targetSize == srcSize;
788
789 if (narrowingConversion)
790 return targetSize <= srcSize;
791 return targetSize >= srcSize;
792}
793
794/// Checks if `dataLayout` describes a little endian layout.
795static bool isBigEndian(const DataLayout &dataLayout) {
796 auto endiannessStr = dyn_cast_or_null<StringAttr>(Val: dataLayout.getEndianness());
797 return endiannessStr && endiannessStr == "big";
798}
799
800/// Converts a value to an integer type of the same size.
801/// Assumes that the type can be converted.
802static Value castToSameSizedInt(OpBuilder &builder, Location loc, Value val,
803 const DataLayout &dataLayout) {
804 Type type = val.getType();
805 assert(isSupportedTypeForConversion(type) &&
806 "expected value to have a convertible type");
807
808 if (isa<IntegerType>(Val: type))
809 return val;
810
811 uint64_t typeBitSize = dataLayout.getTypeSizeInBits(t: type);
812 IntegerType valueSizeInteger = builder.getIntegerType(width: typeBitSize);
813
814 if (isa<LLVM::LLVMPointerType>(Val: type))
815 return builder.createOrFold<LLVM::PtrToIntOp>(location: loc, args&: valueSizeInteger, args&: val);
816 return builder.createOrFold<LLVM::BitcastOp>(location: loc, args&: valueSizeInteger, args&: val);
817}
818
819/// Converts a value with an integer type to `targetType`.
820static Value castIntValueToSameSizedType(OpBuilder &builder, Location loc,
821 Value val, Type targetType) {
822 assert(isa<IntegerType>(val.getType()) &&
823 "expected value to have an integer type");
824 assert(isSupportedTypeForConversion(targetType) &&
825 "expected the target type to be supported for conversions");
826 if (val.getType() == targetType)
827 return val;
828 if (isa<LLVM::LLVMPointerType>(Val: targetType))
829 return builder.createOrFold<LLVM::IntToPtrOp>(location: loc, args&: targetType, args&: val);
830 return builder.createOrFold<LLVM::BitcastOp>(location: loc, args&: targetType, args&: val);
831}
832
833/// Constructs operations that convert `srcValue` into a new value of type
834/// `targetType`. Assumes the types have the same bitsize.
835static Value castSameSizedTypes(OpBuilder &builder, Location loc,
836 Value srcValue, Type targetType,
837 const DataLayout &dataLayout) {
838 Type srcType = srcValue.getType();
839 assert(areConversionCompatible(dataLayout, targetType, srcType,
840 /*narrowingConversion=*/true) &&
841 "expected that the compatibility was checked before");
842
843 // Nothing has to be done if the types are already the same.
844 if (srcType == targetType)
845 return srcValue;
846
847 // In the special case of casting one pointer to another, we want to generate
848 // an address space cast. Bitcasts of pointers are not allowed and using
849 // pointer to integer conversions are not equivalent due to the loss of
850 // provenance.
851 if (isa<LLVM::LLVMPointerType>(Val: targetType) &&
852 isa<LLVM::LLVMPointerType>(Val: srcType))
853 return builder.createOrFold<LLVM::AddrSpaceCastOp>(location: loc, args&: targetType,
854 args&: srcValue);
855
856 // For all other castable types, casting through integers is necessary.
857 Value replacement = castToSameSizedInt(builder, loc, val: srcValue, dataLayout);
858 return castIntValueToSameSizedType(builder, loc, val: replacement, targetType);
859}
860
861/// Constructs operations that convert `srcValue` into a new value of type
862/// `targetType`. Performs bit-level extraction if the source type is larger
863/// than the target type. Assumes that this conversion is possible.
864static Value createExtractAndCast(OpBuilder &builder, Location loc,
865 Value srcValue, Type targetType,
866 const DataLayout &dataLayout) {
867 // Get the types of the source and target values.
868 Type srcType = srcValue.getType();
869 assert(areConversionCompatible(dataLayout, targetType, srcType,
870 /*narrowingConversion=*/true) &&
871 "expected that the compatibility was checked before");
872
873 // Nothing has to be done if the types are already the same. This also
874 // avoids querying the bit size of scalable vector types below.
875 if (srcType == targetType)
876 return srcValue;
877
878 uint64_t srcTypeSize = dataLayout.getTypeSizeInBits(t: srcType);
879 uint64_t targetTypeSize = dataLayout.getTypeSizeInBits(t: targetType);
880 if (srcTypeSize == targetTypeSize)
881 return castSameSizedTypes(builder, loc, srcValue, targetType, dataLayout);
882
883 // First, cast the value to a same-sized integer type.
884 Value replacement = castToSameSizedInt(builder, loc, val: srcValue, dataLayout);
885
886 // Truncate the integer if the size of the target is less than the value.
887 if (isBigEndian(dataLayout)) {
888 uint64_t shiftAmount = srcTypeSize - targetTypeSize;
889 auto shiftConstant = LLVM::ConstantOp::create(
890 builder, location: loc, value: builder.getIntegerAttr(type: srcType, value: shiftAmount));
891 replacement =
892 builder.createOrFold<LLVM::LShrOp>(location: loc, args&: srcValue, args&: shiftConstant);
893 }
894
895 replacement = LLVM::TruncOp::create(
896 builder, location: loc, res: builder.getIntegerType(width: targetTypeSize), arg: replacement);
897
898 // Now cast the integer to the actual target type if required.
899 return castIntValueToSameSizedType(builder, loc, val: replacement, targetType);
900}
901
902/// Constructs operations that insert the bits of `srcValue` into the
903/// "beginning" of `reachingDef` (beginning is endianness dependent).
904/// Assumes that this conversion is possible.
905static Value createInsertAndCast(OpBuilder &builder, Location loc,
906 Value srcValue, Value reachingDef,
907 const DataLayout &dataLayout) {
908
909 assert(areConversionCompatible(dataLayout, reachingDef.getType(),
910 srcValue.getType(),
911 /*narrowingConversion=*/false) &&
912 "expected that the compatibility was checked before");
913
914 // Nothing has to be done if the types are already the same. This also
915 // avoids querying the bit size of scalable vector types below.
916 if (srcValue.getType() == reachingDef.getType())
917 return srcValue;
918
919 uint64_t valueTypeSize = dataLayout.getTypeSizeInBits(t: srcValue.getType());
920 uint64_t slotTypeSize = dataLayout.getTypeSizeInBits(t: reachingDef.getType());
921 if (slotTypeSize == valueTypeSize)
922 return castSameSizedTypes(builder, loc, srcValue, targetType: reachingDef.getType(),
923 dataLayout);
924
925 // In the case where the store only overwrites parts of the memory,
926 // bit fiddling is required to construct the new value.
927
928 // First convert both values to integers of the same size.
929 Value defAsInt = castToSameSizedInt(builder, loc, val: reachingDef, dataLayout);
930 Value valueAsInt = castToSameSizedInt(builder, loc, val: srcValue, dataLayout);
931 // Extend the value to the size of the reaching definition.
932 valueAsInt =
933 builder.createOrFold<LLVM::ZExtOp>(location: loc, args: defAsInt.getType(), args&: valueAsInt);
934 uint64_t sizeDifference = slotTypeSize - valueTypeSize;
935 if (isBigEndian(dataLayout)) {
936 // On big endian systems, a store to the base pointer overwrites the most
937 // significant bits. To accomodate for this, the stored value needs to be
938 // shifted into the according position.
939 Value bigEndianShift = LLVM::ConstantOp::create(
940 builder, location: loc,
941 value: builder.getIntegerAttr(type: defAsInt.getType(), value: sizeDifference));
942 valueAsInt =
943 builder.createOrFold<LLVM::ShlOp>(location: loc, args&: valueAsInt, args&: bigEndianShift);
944 }
945
946 // Construct the mask that is used to erase the bits that are overwritten by
947 // the store.
948 APInt maskValue;
949 if (isBigEndian(dataLayout)) {
950 // Build a mask that has the most significant bits set to zero.
951 // Note: This is the same as 2^sizeDifference - 1
952 maskValue = APInt::getAllOnes(numBits: sizeDifference).zext(width: slotTypeSize);
953 } else {
954 // Build a mask that has the least significant bits set to zero.
955 // Note: This is the same as -(2^valueTypeSize)
956 maskValue = APInt::getAllOnes(numBits: valueTypeSize).zext(width: slotTypeSize);
957 maskValue.flipAllBits();
958 }
959
960 // Mask out the affected bits ...
961 Value mask = LLVM::ConstantOp::create(
962 builder, location: loc, value: builder.getIntegerAttr(type: defAsInt.getType(), value: maskValue));
963 Value masked = builder.createOrFold<LLVM::AndOp>(location: loc, args&: defAsInt, args&: mask);
964
965 // ... and combine the result with the new value.
966 Value combined = builder.createOrFold<LLVM::OrOp>(location: loc, args&: masked, args&: valueAsInt);
967
968 return castIntValueToSameSizedType(builder, loc, val: combined,
969 targetType: reachingDef.getType());
970}
971
972Value LLVM::StoreOp::getStored(const MemorySlot &slot, OpBuilder &builder,
973 Value reachingDef,
974 const DataLayout &dataLayout) {
975 assert(reachingDef && reachingDef.getType() == slot.valueType &&
976 "expected the reaching definition's type to match the slot's type");
977 return createInsertAndCast(builder, loc: getLoc(), srcValue: getValue(), reachingDef,
978 dataLayout);
979}
980
981bool LLVM::LoadOp::canUsesBeRemoved(
982 const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
983 SmallVectorImpl<OpOperand *> &newBlockingUses,
984 const DataLayout &dataLayout) {
985 if (blockingUses.size() != 1)
986 return false;
987 Value blockingUse = (*blockingUses.begin())->get();
988 // If the blocking use is the slot ptr itself, there will be enough
989 // context to reconstruct the result of the load at removal time, so it can
990 // be removed (provided it is not volatile).
991 return blockingUse == slot.ptr && getAddr() == slot.ptr &&
992 areConversionCompatible(layout: dataLayout, targetType: getResult().getType(),
993 srcType: slot.valueType,
994 /*narrowingConversion=*/true) &&
995 !getVolatile_();
996}
997
998DeletionKind LLVM::LoadOp::removeBlockingUses(
999 const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
1000 OpBuilder &builder, Value reachingDefinition,
1001 const DataLayout &dataLayout) {
1002 // `canUsesBeRemoved` checked this blocking use must be the loaded slot
1003 // pointer.
1004 Value newResult = createExtractAndCast(builder, loc: getLoc(), srcValue: reachingDefinition,
1005 targetType: getResult().getType(), dataLayout);
1006 getResult().replaceAllUsesWith(newValue: newResult);
1007 return DeletionKind::Delete;
1008}
1009
1010bool LLVM::StoreOp::canUsesBeRemoved(
1011 const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
1012 SmallVectorImpl<OpOperand *> &newBlockingUses,
1013 const DataLayout &dataLayout) {
1014 if (blockingUses.size() != 1)
1015 return false;
1016 Value blockingUse = (*blockingUses.begin())->get();
1017 // If the blocking use is the slot ptr itself, dropping the store is
1018 // fine, provided we are currently promoting its target value. Don't allow a
1019 // store OF the slot pointer, only INTO the slot pointer.
1020 return blockingUse == slot.ptr && getAddr() == slot.ptr &&
1021 getValue() != slot.ptr &&
1022 areConversionCompatible(layout: dataLayout, targetType: slot.valueType,
1023 srcType: getValue().getType(),
1024 /*narrowingConversion=*/false) &&
1025 !getVolatile_();
1026}
1027
1028DeletionKind LLVM::StoreOp::removeBlockingUses(
1029 const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
1030 OpBuilder &builder, Value reachingDefinition,
1031 const DataLayout &dataLayout) {
1032 return DeletionKind::Delete;
1033}
1034
1035/// Checks if `slot` can be accessed through the provided access type.
1036static bool isValidAccessType(const MemorySlot &slot, Type accessType,
1037 const DataLayout &dataLayout) {
1038 return dataLayout.getTypeSize(t: accessType) <=
1039 dataLayout.getTypeSize(t: slot.valueType);
1040}
1041
1042LogicalResult LLVM::LoadOp::ensureOnlySafeAccesses(
1043 const MemorySlot &slot, SmallVectorImpl<MemorySlot> &mustBeSafelyUsed,
1044 const DataLayout &dataLayout) {
1045 return success(IsSuccess: getAddr() != slot.ptr ||
1046 isValidAccessType(slot, accessType: getType(), dataLayout));
1047}
1048
1049LogicalResult LLVM::StoreOp::ensureOnlySafeAccesses(
1050 const MemorySlot &slot, SmallVectorImpl<MemorySlot> &mustBeSafelyUsed,
1051 const DataLayout &dataLayout) {
1052 return success(IsSuccess: getAddr() != slot.ptr ||
1053 isValidAccessType(slot, accessType: getValue().getType(), dataLayout));
1054}
1055
1056/// Returns the subslot's type at the requested index.
1057static Type getTypeAtIndex(const DestructurableMemorySlot &slot,
1058 Attribute index) {
1059 auto subelementIndexMap =
1060 cast<DestructurableTypeInterface>(Val: slot.valueType).getSubelementIndexMap();
1061 if (!subelementIndexMap)
1062 return {};
1063 assert(!subelementIndexMap->empty());
1064
1065 // Note: Returns a null-type when no entry was found.
1066 return subelementIndexMap->lookup(Val: index);
1067}
1068
1069bool LLVM::LoadOp::canRewire(const DestructurableMemorySlot &slot,
1070 SmallPtrSetImpl<Attribute> &usedIndices,
1071 SmallVectorImpl<MemorySlot> &mustBeSafelyUsed,
1072 const DataLayout &dataLayout) {
1073 if (getVolatile_())
1074 return false;
1075
1076 // A load always accesses the first element of the destructured slot.
1077 auto index = IntegerAttr::get(type: IntegerType::get(context: getContext(), width: 32), value: 0);
1078 Type subslotType = getTypeAtIndex(slot, index);
1079 if (!subslotType)
1080 return false;
1081
1082 // The access can only be replaced when the subslot is read within its bounds.
1083 if (dataLayout.getTypeSize(t: getType()) > dataLayout.getTypeSize(t: subslotType))
1084 return false;
1085
1086 usedIndices.insert(Ptr: index);
1087 return true;
1088}
1089
1090DeletionKind LLVM::LoadOp::rewire(const DestructurableMemorySlot &slot,
1091 DenseMap<Attribute, MemorySlot> &subslots,
1092 OpBuilder &builder,
1093 const DataLayout &dataLayout) {
1094 auto index = IntegerAttr::get(type: IntegerType::get(context: getContext(), width: 32), value: 0);
1095 auto it = subslots.find(Val: index);
1096 assert(it != subslots.end());
1097
1098 getAddrMutable().set(it->getSecond().ptr);
1099 return DeletionKind::Keep;
1100}
1101
1102bool LLVM::StoreOp::canRewire(const DestructurableMemorySlot &slot,
1103 SmallPtrSetImpl<Attribute> &usedIndices,
1104 SmallVectorImpl<MemorySlot> &mustBeSafelyUsed,
1105 const DataLayout &dataLayout) {
1106 if (getVolatile_())
1107 return false;
1108
1109 // Storing the pointer to memory cannot be dealt with.
1110 if (getValue() == slot.ptr)
1111 return false;
1112
1113 // A store always accesses the first element of the destructured slot.
1114 auto index = IntegerAttr::get(type: IntegerType::get(context: getContext(), width: 32), value: 0);
1115 Type subslotType = getTypeAtIndex(slot, index);
1116 if (!subslotType)
1117 return false;
1118
1119 // The access can only be replaced when the subslot is read within its bounds.
1120 if (dataLayout.getTypeSize(t: getValue().getType()) >
1121 dataLayout.getTypeSize(t: subslotType))
1122 return false;
1123
1124 usedIndices.insert(Ptr: index);
1125 return true;
1126}
1127
1128DeletionKind LLVM::StoreOp::rewire(const DestructurableMemorySlot &slot,
1129 DenseMap<Attribute, MemorySlot> &subslots,
1130 OpBuilder &builder,
1131 const DataLayout &dataLayout) {
1132 auto index = IntegerAttr::get(type: IntegerType::get(context: getContext(), width: 32), value: 0);
1133 auto it = subslots.find(Val: index);
1134 assert(it != subslots.end());
1135
1136 getAddrMutable().set(it->getSecond().ptr);
1137 return DeletionKind::Keep;
1138}
1139
1140//===----------------------------------------------------------------------===//
1141// Interfaces for discardable OPs
1142//===----------------------------------------------------------------------===//
1143
1144/// Conditions the deletion of the operation to the removal of all its uses.
1145bool mlir::LLVM::forwardToUsers(Operation *op,
1146 SmallVectorImpl<OpOperand *> &newBlockingUses) {
1147 for (Value result : op->getResults())
1148 for (OpOperand &use : result.getUses())
1149 newBlockingUses.push_back(Elt: &use);
1150 return true;
1151}
1152
1153//===----------------------------------------------------------------------===//
1154// Interfaces for GEPOp
1155//===----------------------------------------------------------------------===//
1156
1157static bool hasAllZeroIndices(LLVM::GEPOp gepOp) {
1158 return llvm::all_of(Range: gepOp.getIndices(), P: [](auto index) {
1159 auto indexAttr = llvm::dyn_cast_if_present<IntegerAttr>(index);
1160 return indexAttr && indexAttr.getValue() == 0;
1161 });
1162}
1163
1164bool LLVM::GEPOp::canUsesBeRemoved(
1165 const SmallPtrSetImpl<OpOperand *> &blockingUses,
1166 SmallVectorImpl<OpOperand *> &newBlockingUses,
1167 const DataLayout &dataLayout) {
1168 // GEP can be removed as long as it is a no-op and its users can be removed.
1169 // `inrange` is only valid on constant GEP expressions, so an inrange GEP on
1170 // an alloca is illegal and we bail out.
1171 if (getInrangeAttr() || !hasAllZeroIndices(gepOp: *this))
1172 return false;
1173 return forwardToUsers(op: *this, newBlockingUses);
1174}
1175
1176DeletionKind LLVM::GEPOp::removeBlockingUses(
1177 const SmallPtrSetImpl<OpOperand *> &blockingUses, OpBuilder &builder) {
1178 return DeletionKind::Delete;
1179}
1180
1181/// Returns the amount of bytes the provided GEP elements will offset the
1182/// pointer by. Returns nullopt if no constant offset could be computed.
1183static std::optional<uint64_t> gepToByteOffset(const DataLayout &dataLayout,
1184 LLVM::GEPOp gep) {
1185 // Collects all indices.
1186 SmallVector<uint64_t> indices;
1187 for (auto index : gep.getIndices()) {
1188 auto constIndex = dyn_cast<IntegerAttr>(Val&: index);
1189 if (!constIndex)
1190 return {};
1191 int64_t gepIndex = constIndex.getInt();
1192 // Negative indices are not supported.
1193 if (gepIndex < 0)
1194 return {};
1195 indices.push_back(Elt: gepIndex);
1196 }
1197
1198 Type currentType = gep.getElemType();
1199 uint64_t offset = indices[0] * dataLayout.getTypeSize(t: currentType);
1200
1201 for (uint64_t index : llvm::drop_begin(RangeOrContainer&: indices)) {
1202 bool shouldCancel =
1203 TypeSwitch<Type, bool>(currentType)
1204 .Case(caseFn: [&](LLVM::LLVMArrayType arrayType) {
1205 offset +=
1206 index * dataLayout.getTypeSize(t: arrayType.getElementType());
1207 currentType = arrayType.getElementType();
1208 return false;
1209 })
1210 .Case(caseFn: [&](LLVM::LLVMStructType structType) {
1211 offset += structType.getElementOffset(dataLayout, index);
1212 currentType = structType.getBody()[index];
1213 return false;
1214 })
1215 .Default(defaultFn: [&](Type type) {
1216 LDBG() << "[sroa] Unsupported type for offset computations"
1217 << type;
1218 return true;
1219 });
1220
1221 if (shouldCancel)
1222 return std::nullopt;
1223 }
1224
1225 return offset;
1226}
1227
1228namespace {
1229/// A struct that stores both the index into the aggregate type of the slot as
1230/// well as the corresponding byte offset in memory.
1231struct SubslotAccessInfo {
1232 /// The parent slot's index that the access falls into.
1233 uint32_t index;
1234 /// The offset into the subslot of the access.
1235 uint64_t subslotOffset;
1236};
1237} // namespace
1238
1239/// Computes subslot access information for an access into `slot` with the given
1240/// offset.
1241/// Returns nullopt when the offset is out-of-bounds or when the access is into
1242/// the padding of `slot`.
1243static std::optional<SubslotAccessInfo>
1244getSubslotAccessInfo(const DestructurableMemorySlot &slot,
1245 const DataLayout &dataLayout, LLVM::GEPOp gep) {
1246 std::optional<uint64_t> offset = gepToByteOffset(dataLayout, gep);
1247 if (!offset)
1248 return {};
1249
1250 // Helper to check that a constant index is in the bounds of the GEP index
1251 // representation. LLVM dialects's GEP arguments have a limited bitwidth, thus
1252 // this additional check is necessary.
1253 auto isOutOfBoundsGEPIndex = [](uint64_t index) {
1254 return index >= (1 << LLVM::kGEPConstantBitWidth);
1255 };
1256
1257 Type type = slot.valueType;
1258 if (*offset >= dataLayout.getTypeSize(t: type))
1259 return {};
1260 return TypeSwitch<Type, std::optional<SubslotAccessInfo>>(type)
1261 .Case(caseFn: [&](LLVM::LLVMArrayType arrayType)
1262 -> std::optional<SubslotAccessInfo> {
1263 // Find which element of the array contains the offset.
1264 uint64_t elemSize = dataLayout.getTypeSize(t: arrayType.getElementType());
1265 uint64_t index = *offset / elemSize;
1266 if (isOutOfBoundsGEPIndex(index))
1267 return {};
1268 return SubslotAccessInfo{.index: static_cast<uint32_t>(index),
1269 .subslotOffset: *offset - (index * elemSize)};
1270 })
1271 .Case(caseFn: [&](LLVM::LLVMStructType structType)
1272 -> std::optional<SubslotAccessInfo> {
1273 // Walk over the elements of the struct to find in which of
1274 // them the offset is.
1275 for (auto [index, elem] : llvm::enumerate(First: structType.getBody())) {
1276 uint64_t elemOffset = structType.getElementOffset(dataLayout, index);
1277 // If the offset is in padding, cancel the rewrite.
1278 if (*offset < elemOffset)
1279 return {};
1280
1281 if (*offset < elemOffset + dataLayout.getTypeSize(t: elem)) {
1282 if (isOutOfBoundsGEPIndex(index))
1283 return {};
1284 // The offset is within this element, stop iterating the
1285 // struct and return the index.
1286 return SubslotAccessInfo{.index: static_cast<uint32_t>(index),
1287 .subslotOffset: *offset - elemOffset};
1288 }
1289 }
1290
1291 return {};
1292 });
1293}
1294
1295/// Constructs a byte array type of the given size.
1296static LLVM::LLVMArrayType getByteArrayType(MLIRContext *context,
1297 unsigned size) {
1298 auto byteType = IntegerType::get(context, width: 8);
1299 return LLVM::LLVMArrayType::get(context, elementType: byteType, numElements: size);
1300}
1301
1302LogicalResult LLVM::GEPOp::ensureOnlySafeAccesses(
1303 const MemorySlot &slot, SmallVectorImpl<MemorySlot> &mustBeSafelyUsed,
1304 const DataLayout &dataLayout) {
1305 if (getBase() != slot.ptr)
1306 return success();
1307 std::optional<uint64_t> gepOffset = gepToByteOffset(dataLayout, gep: *this);
1308 if (!gepOffset)
1309 return failure();
1310 uint64_t slotSize = dataLayout.getTypeSize(t: slot.valueType);
1311 // Check that the access is strictly inside the slot.
1312 if (*gepOffset >= slotSize)
1313 return failure();
1314 // Every access that remains in bounds of the remaining slot is considered
1315 // legal.
1316 mustBeSafelyUsed.emplace_back<MemorySlot>(
1317 Args: {.ptr: getRes(), .valueType: getByteArrayType(context: getContext(), size: slotSize - *gepOffset)});
1318 return success();
1319}
1320
1321bool LLVM::GEPOp::canRewire(const DestructurableMemorySlot &slot,
1322 SmallPtrSetImpl<Attribute> &usedIndices,
1323 SmallVectorImpl<MemorySlot> &mustBeSafelyUsed,
1324 const DataLayout &dataLayout) {
1325 if (!isa<LLVM::LLVMPointerType>(Val: getBase().getType()))
1326 return false;
1327
1328 if (getBase() != slot.ptr)
1329 return false;
1330 // `inrange` is only valid on constant GEP expressions, so an inrange GEP on
1331 // an alloca is illegal and SROA bails out.
1332 if (getInrangeAttr())
1333 return false;
1334 std::optional<SubslotAccessInfo> accessInfo =
1335 getSubslotAccessInfo(slot, dataLayout, gep: *this);
1336 if (!accessInfo)
1337 return false;
1338 auto indexAttr =
1339 IntegerAttr::get(type: IntegerType::get(context: getContext(), width: 32), value: accessInfo->index);
1340 assert(slot.subelementTypes.contains(indexAttr));
1341 usedIndices.insert(Ptr: indexAttr);
1342
1343 // The remainder of the subslot should be accesses in-bounds. Thus, we create
1344 // a dummy slot with the size of the remainder.
1345 Type subslotType = slot.subelementTypes.lookup(Val: indexAttr);
1346 uint64_t slotSize = dataLayout.getTypeSize(t: subslotType);
1347 LLVM::LLVMArrayType remainingSlotType =
1348 getByteArrayType(context: getContext(), size: slotSize - accessInfo->subslotOffset);
1349 mustBeSafelyUsed.emplace_back<MemorySlot>(Args: {.ptr: getRes(), .valueType: remainingSlotType});
1350
1351 return true;
1352}
1353
1354DeletionKind LLVM::GEPOp::rewire(const DestructurableMemorySlot &slot,
1355 DenseMap<Attribute, MemorySlot> &subslots,
1356 OpBuilder &builder,
1357 const DataLayout &dataLayout) {
1358 std::optional<SubslotAccessInfo> accessInfo =
1359 getSubslotAccessInfo(slot, dataLayout, gep: *this);
1360 assert(accessInfo && "expected access info to be checked before");
1361 auto indexAttr =
1362 IntegerAttr::get(type: IntegerType::get(context: getContext(), width: 32), value: accessInfo->index);
1363 const MemorySlot &newSlot = subslots.at(Val: indexAttr);
1364
1365 auto byteType = IntegerType::get(context: builder.getContext(), width: 8);
1366 auto newPtr = builder.createOrFold<LLVM::GEPOp>(
1367 location: getLoc(), args: getResult().getType(), args&: byteType, args: newSlot.ptr,
1368 args: ArrayRef<GEPArg>(accessInfo->subslotOffset), args: getNoWrapFlags());
1369 getResult().replaceAllUsesWith(newValue: newPtr);
1370 return DeletionKind::Delete;
1371}
1372
1373SmallVector<Value> mlir::LLVM::AtomicCmpXchgOp::getAccessedOperands() {
1374 return {getPtr()};
1375}
1376
1377SmallVector<Value> mlir::LLVM::AtomicRMWOp::getAccessedOperands() {
1378 return {getPtr()};
1379}
1380
1381SmallVector<Value> mlir::LLVM::LoadOp::getAccessedOperands() {
1382 return {getAddr()};
1383}
1384
1385SmallVector<Value> mlir::LLVM::StoreOp::getAccessedOperands() {
1386 return {getAddr()};
1387}
1388
1389#define GET_OP_CLASSES
1390#include "mlir/Dialect/LLVMIR/LLVMMemoryOps.cpp.inc"
1391