| 1 | //===- LLVMVectorIntrinsicOps.cpp - LLVM dialect vector intrinsics --------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file defines the ops of op partition `VectorIntrinsic` of the LLVM |
| 10 | // dialect. |
| 11 | // |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #include "mlir/Dialect/LLVMIR/LLVMVectorIntrinsicOps.h" |
| 15 | |
| 16 | #include "IR/LLVMOps.h" |
| 17 | #include "mlir/IR/Builders.h" |
| 18 | #include "mlir/IR/DialectImplementation.h" |
| 19 | |
| 20 | using namespace mlir; |
| 21 | using namespace mlir::LLVM; |
| 22 | |
| 23 | static ArrayAttr getLLVMAlignParamForCompressExpand(OpBuilder &builder, |
| 24 | bool isExpandLoad, |
| 25 | uint64_t alignment = 1) { |
| 26 | // From |
| 27 | // https://llvm.org/docs/LangRef.html#llvm-masked-expandload-intrinsics |
| 28 | // https://llvm.org/docs/LangRef.html#llvm-masked-compressstore-intrinsics |
| 29 | // |
| 30 | // The pointer alignment defaults to 1. |
| 31 | if (alignment == 1) { |
| 32 | return nullptr; |
| 33 | } |
| 34 | |
| 35 | auto emptyDictAttr = builder.getDictionaryAttr(value: {}); |
| 36 | auto alignmentAttr = builder.getI64IntegerAttr(value: alignment); |
| 37 | auto namedAttr = |
| 38 | builder.getNamedAttr(name: LLVMDialect::getAlignAttrName(), val: alignmentAttr); |
| 39 | SmallVector<mlir::NamedAttribute> attrs = {namedAttr}; |
| 40 | auto alignDictAttr = builder.getDictionaryAttr(value: attrs); |
| 41 | // From |
| 42 | // https://llvm.org/docs/LangRef.html#llvm-masked-expandload-intrinsics |
| 43 | // https://llvm.org/docs/LangRef.html#llvm-masked-compressstore-intrinsics |
| 44 | // |
| 45 | // The align parameter attribute can be provided for [expandload]'s first |
| 46 | // argument. The align parameter attribute can be provided for |
| 47 | // [compressstore]'s second argument. |
| 48 | int pos = isExpandLoad ? 0 : 1; |
| 49 | return pos == 0 ? builder.getArrayAttr( |
| 50 | value: {alignDictAttr, emptyDictAttr, emptyDictAttr}) |
| 51 | : builder.getArrayAttr( |
| 52 | value: {emptyDictAttr, alignDictAttr, emptyDictAttr}); |
| 53 | } |
| 54 | |
| 55 | //===----------------------------------------------------------------------===// |
| 56 | // masked_gather (intrinsic) |
| 57 | //===----------------------------------------------------------------------===// |
| 58 | |
| 59 | LogicalResult LLVM::masked_gather::verify() { |
| 60 | auto ptrsVectorType = getPtrs().getType(); |
| 61 | Type expectedPtrsVectorType = |
| 62 | LLVM::getVectorType(elementType: extractVectorElementType(type: ptrsVectorType), |
| 63 | numElements: LLVM::getVectorNumElements(type: getRes().getType())); |
| 64 | // Vector of pointers type should match result vector type, other than the |
| 65 | // element type. |
| 66 | if (ptrsVectorType != expectedPtrsVectorType) |
| 67 | return emitOpError(message: "expected operand #1 type to be " ) |
| 68 | << expectedPtrsVectorType; |
| 69 | return success(); |
| 70 | } |
| 71 | |
| 72 | //===----------------------------------------------------------------------===// |
| 73 | // masked_scatter (intrinsic) |
| 74 | //===----------------------------------------------------------------------===// |
| 75 | |
| 76 | LogicalResult LLVM::masked_scatter::verify() { |
| 77 | auto ptrsVectorType = getPtrs().getType(); |
| 78 | Type expectedPtrsVectorType = |
| 79 | LLVM::getVectorType(elementType: extractVectorElementType(type: ptrsVectorType), |
| 80 | numElements: LLVM::getVectorNumElements(type: getValue().getType())); |
| 81 | // Vector of pointers type should match value vector type, other than the |
| 82 | // element type. |
| 83 | if (ptrsVectorType != expectedPtrsVectorType) |
| 84 | return emitOpError(message: "expected operand #2 type to be " ) |
| 85 | << expectedPtrsVectorType; |
| 86 | return success(); |
| 87 | } |
| 88 | |
| 89 | //===----------------------------------------------------------------------===// |
| 90 | // masked_expandload (intrinsic) |
| 91 | //===----------------------------------------------------------------------===// |
| 92 | |
| 93 | void LLVM::masked_expandload::build(OpBuilder &builder, OperationState &state, |
| 94 | mlir::TypeRange resTys, Value ptr, |
| 95 | Value mask, Value passthru, |
| 96 | uint64_t align) { |
| 97 | ArrayAttr argAttrs = getLLVMAlignParamForCompressExpand(builder, isExpandLoad: true, alignment: align); |
| 98 | build(odsBuilder&: builder, odsState&: state, resultTypes: resTys, ptr, mask, passthru, /*arg_attrs=*/argAttrs, |
| 99 | /*res_attrs=*/nullptr); |
| 100 | } |
| 101 | |
| 102 | //===----------------------------------------------------------------------===// |
| 103 | // masked_compressstore (intrinsic) |
| 104 | //===----------------------------------------------------------------------===// |
| 105 | |
| 106 | void LLVM::masked_compressstore::build(OpBuilder &builder, |
| 107 | OperationState &state, Value value, |
| 108 | Value ptr, Value mask, uint64_t align) { |
| 109 | ArrayAttr argAttrs = |
| 110 | getLLVMAlignParamForCompressExpand(builder, isExpandLoad: false, alignment: align); |
| 111 | build(odsBuilder&: builder, odsState&: state, value, ptr, mask, /*arg_attrs=*/argAttrs, |
| 112 | /*res_attrs=*/nullptr); |
| 113 | } |
| 114 | |
| 115 | SmallVector<Value> mlir::LLVM::MaskedLoadOp::getAccessedOperands() { |
| 116 | return {getData()}; |
| 117 | } |
| 118 | |
| 119 | SmallVector<Value> mlir::LLVM::MaskedStoreOp::getAccessedOperands() { |
| 120 | return {getData()}; |
| 121 | } |
| 122 | |
| 123 | SmallVector<Value> mlir::LLVM::masked_gather::getAccessedOperands() { |
| 124 | return {getPtrs()}; |
| 125 | } |
| 126 | |
| 127 | SmallVector<Value> mlir::LLVM::masked_scatter::getAccessedOperands() { |
| 128 | return {getPtrs()}; |
| 129 | } |
| 130 | |
| 131 | #define GET_OP_CLASSES |
| 132 | #include "mlir/Dialect/LLVMIR/LLVMVectorIntrinsicOps.cpp.inc" |
| 133 | |