| 1 | //===- MPIToLLVM.cpp - MPI to LLVM dialect conversion ---------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | // |
| 10 | // Copyright (C) by Argonne National Laboratory |
| 11 | // See COPYRIGHT in top-level directory |
| 12 | // of MPICH source repository. |
| 13 | // |
| 14 | |
| 15 | #include "mlir/Conversion/MPIToLLVM/MPIToLLVM.h" |
| 16 | #include "mlir/Conversion/ConvertToLLVM/ToLLVMInterface.h" |
| 17 | #include "mlir/Conversion/LLVMCommon/Pattern.h" |
| 18 | #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h" |
| 19 | #include "mlir/Dialect/DLTI/DLTI.h" |
| 20 | #include "mlir/Dialect/LLVMIR/LLVMAggregateOps.h" |
| 21 | #include "mlir/Dialect/LLVMIR/LLVMArithOps.h" |
| 22 | #include "mlir/Dialect/LLVMIR/LLVMCallOps.h" |
| 23 | #include "mlir/Dialect/LLVMIR/LLVMConstantOps.h" |
| 24 | #include "mlir/Dialect/LLVMIR/LLVMConversionOps.h" |
| 25 | #include "mlir/Dialect/LLVMIR/LLVMMemoryOps.h" |
| 26 | #include "mlir/Dialect/LLVMIR/LLVMModuleOps.h" |
| 27 | #include "mlir/Dialect/LLVMIR/LLVMTypes.h" |
| 28 | #include "mlir/Dialect/MPI/IR/MPI.h" |
| 29 | #include "mlir/Dialect/MPI/IR/Utils.h" |
| 30 | #include "mlir/Transforms/DialectConversion.h" |
| 31 | #include <memory> |
| 32 | |
| 33 | using namespace mlir; |
| 34 | |
| 35 | namespace { |
| 36 | |
| 37 | template <typename Op, typename... Args> |
| 38 | static Op getOrDefineGlobal(ModuleOp &moduleOp, const Location loc, |
| 39 | ConversionPatternRewriter &rewriter, StringRef name, |
| 40 | Args &&...args) { |
| 41 | Op ret; |
| 42 | if (!(ret = moduleOp.lookupSymbol<Op>(name))) { |
| 43 | ConversionPatternRewriter::InsertionGuard guard(rewriter); |
| 44 | rewriter.setInsertionPointToStart(moduleOp.getBody()); |
| 45 | ret = Op::create(rewriter, loc, std::forward<Args>(args)...); |
| 46 | } |
| 47 | return ret; |
| 48 | } |
| 49 | |
| 50 | static LLVM::LLVMFuncOp getOrDefineFunction(ModuleOp &moduleOp, |
| 51 | const Location loc, |
| 52 | ConversionPatternRewriter &rewriter, |
| 53 | StringRef name, |
| 54 | LLVM::LLVMFunctionType type) { |
| 55 | return getOrDefineGlobal<LLVM::LLVMFuncOp>( |
| 56 | moduleOp, loc, rewriter, name, args&: name, args&: type, args: LLVM::Linkage::External); |
| 57 | } |
| 58 | |
| 59 | std::pair<Value, Value> getRawPtrAndSize(const Location loc, |
| 60 | ConversionPatternRewriter &rewriter, |
| 61 | Value memRef, int64_t rank, |
| 62 | Type elType) { |
| 63 | Type ptrType = LLVM::LLVMPointerType::get(context: rewriter.getContext()); |
| 64 | Type i32Type = rewriter.getI32Type(); |
| 65 | auto descriptorType = cast<LLVM::LLVMStructType>(Val: memRef.getType()); |
| 66 | // The offset and the sizes of a memref descriptor have the converted index |
| 67 | // type, which is not necessarily `i64`. Take it from the descriptor itself. |
| 68 | auto indexType = cast<IntegerType>(Val: descriptorType.getBody()[2]); |
| 69 | |
| 70 | Value dataPtr = |
| 71 | LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, res: ptrType, container: memRef, position: 1); |
| 72 | Value offset = |
| 73 | LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, res: indexType, container: memRef, position: 2); |
| 74 | Value resPtr = |
| 75 | LLVM::GEPOp::create(builder&: rewriter, location: loc, resultType: ptrType, elementType: elType, basePtr: dataPtr, indices: offset); |
| 76 | Value size = LLVM::ConstantOp::create(builder&: rewriter, location: loc, res: i32Type, |
| 77 | value: rewriter.getI32IntegerAttr(value: 1)); |
| 78 | if (descriptorType.getBody().size() > 3) { |
| 79 | for (int64_t i = 0; i < rank; ++i) { |
| 80 | Value dim = LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: memRef, |
| 81 | position: ArrayRef<int64_t>{3, i}); |
| 82 | // The MPI interface counts elements in an `i32`, so adjust the |
| 83 | // index-typed extent to that width. Extents are non-negative, hence the |
| 84 | // zero extension. |
| 85 | if (indexType.getWidth() > 32) |
| 86 | dim = LLVM::TruncOp::create(builder&: rewriter, location: loc, res: i32Type, arg: dim); |
| 87 | else if (indexType.getWidth() < 32) |
| 88 | dim = LLVM::ZExtOp::create(builder&: rewriter, location: loc, res: i32Type, arg: dim); |
| 89 | size = LLVM::MulOp::create(builder&: rewriter, location: loc, res: i32Type, lhs: dim, rhs: size); |
| 90 | } |
| 91 | } |
| 92 | return {resPtr, size}; |
| 93 | } |
| 94 | |
| 95 | /// When lowering the mpi dialect to functions calls certain details |
| 96 | /// differ between various MPI implementations. This class will provide |
| 97 | /// these in a generic way, depending on the MPI implementation that got |
| 98 | /// selected by the DLTI attribute on the module. |
| 99 | class MPIImplTraits { |
| 100 | ModuleOp &moduleOp; |
| 101 | |
| 102 | public: |
| 103 | /// Instantiate a new MPIImplTraits object according to the DLTI attribute |
| 104 | /// on the given module. Default to MPICH if no attribute is present or |
| 105 | /// the value is unknown. |
| 106 | static std::unique_ptr<MPIImplTraits> get(ModuleOp &moduleOp); |
| 107 | |
| 108 | explicit MPIImplTraits(ModuleOp &moduleOp) : moduleOp(moduleOp) {} |
| 109 | |
| 110 | virtual ~MPIImplTraits() = default; |
| 111 | |
| 112 | ModuleOp &getModuleOp() { return moduleOp; } |
| 113 | |
| 114 | /// Gets or creates MPI_COMM_WORLD as a Value. |
| 115 | /// Different MPI implementations have different communicator types. |
| 116 | /// Using i64 as a portable, intermediate type. |
| 117 | /// Appropriate cast needs to take place before calling MPI functions. |
| 118 | virtual Value getCommWorld(Location loc, |
| 119 | ConversionPatternRewriter &rewriter) = 0; |
| 120 | |
| 121 | /// Type converter provides i64 type for communicator type. |
| 122 | /// Converts to native type, which might be ptr or int or whatever. |
| 123 | virtual Value castComm(Location loc, ConversionPatternRewriter &rewriter, |
| 124 | Value comm) = 0; |
| 125 | |
| 126 | /// Get the MPI_STATUS_IGNORE value (typically a pointer type). |
| 127 | virtual intptr_t getStatusIgnore() = 0; |
| 128 | |
| 129 | /// Get the MPI_IN_PLACE value (void *). |
| 130 | virtual void *getInPlace() = 0; |
| 131 | |
| 132 | /// Gets or creates an MPI datatype as a value which corresponds to the given |
| 133 | /// type. |
| 134 | virtual Value getDataType(Location loc, ConversionPatternRewriter &rewriter, |
| 135 | Type type) = 0; |
| 136 | |
| 137 | /// Gets or creates an MPI_Op value which corresponds to the given |
| 138 | /// enum value. |
| 139 | virtual Value getMPIOp(Location loc, ConversionPatternRewriter &rewriter, |
| 140 | mpi::MPI_ReductionOpEnum opAttr) = 0; |
| 141 | }; |
| 142 | |
| 143 | //===----------------------------------------------------------------------===// |
| 144 | // Implementation details for MPICH ABI compatible MPI implementations |
| 145 | //===----------------------------------------------------------------------===// |
| 146 | |
| 147 | class MPICHImplTraits : public MPIImplTraits { |
| 148 | static constexpr int MPI_FLOAT = 0x4c00040a; |
| 149 | static constexpr int MPI_DOUBLE = 0x4c00080b; |
| 150 | static constexpr int MPI_INT8_T = 0x4c000137; |
| 151 | static constexpr int MPI_INT16_T = 0x4c000238; |
| 152 | static constexpr int MPI_INT32_T = 0x4c000439; |
| 153 | static constexpr int MPI_INT64_T = 0x4c00083a; |
| 154 | static constexpr int MPI_UINT8_T = 0x4c00013b; |
| 155 | static constexpr int MPI_UINT16_T = 0x4c00023c; |
| 156 | static constexpr int MPI_UINT32_T = 0x4c00043d; |
| 157 | static constexpr int MPI_UINT64_T = 0x4c00083e; |
| 158 | static constexpr int MPI_MAX = 0x58000001; |
| 159 | static constexpr int MPI_MIN = 0x58000002; |
| 160 | static constexpr int MPI_SUM = 0x58000003; |
| 161 | static constexpr int MPI_PROD = 0x58000004; |
| 162 | static constexpr int MPI_LAND = 0x58000005; |
| 163 | static constexpr int MPI_BAND = 0x58000006; |
| 164 | static constexpr int MPI_LOR = 0x58000007; |
| 165 | static constexpr int MPI_BOR = 0x58000008; |
| 166 | static constexpr int MPI_LXOR = 0x58000009; |
| 167 | static constexpr int MPI_BXOR = 0x5800000a; |
| 168 | static constexpr int MPI_MINLOC = 0x5800000b; |
| 169 | static constexpr int MPI_MAXLOC = 0x5800000c; |
| 170 | static constexpr int MPI_REPLACE = 0x5800000d; |
| 171 | static constexpr int MPI_NO_OP = 0x5800000e; |
| 172 | |
| 173 | public: |
| 174 | using MPIImplTraits::MPIImplTraits; |
| 175 | |
| 176 | ~MPICHImplTraits() override = default; |
| 177 | |
| 178 | Value getCommWorld(const Location loc, |
| 179 | ConversionPatternRewriter &rewriter) override { |
| 180 | static constexpr int MPI_COMM_WORLD = 0x44000000; |
| 181 | return LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: rewriter.getI64Type(), |
| 182 | value: MPI_COMM_WORLD); |
| 183 | } |
| 184 | |
| 185 | Value castComm(const Location loc, ConversionPatternRewriter &rewriter, |
| 186 | Value comm) override { |
| 187 | return LLVM::TruncOp::create(builder&: rewriter, location: loc, res: rewriter.getI32Type(), arg: comm); |
| 188 | } |
| 189 | |
| 190 | intptr_t getStatusIgnore() override { return 1; } |
| 191 | |
| 192 | void *getInPlace() override { return reinterpret_cast<void *>(-1); } |
| 193 | |
| 194 | Value getDataType(const Location loc, ConversionPatternRewriter &rewriter, |
| 195 | Type type) override { |
| 196 | int32_t mtype = 0; |
| 197 | if (type.isF32()) |
| 198 | mtype = MPI_FLOAT; |
| 199 | else if (type.isF64()) |
| 200 | mtype = MPI_DOUBLE; |
| 201 | else if (type.isInteger(width: 64) && !type.isUnsignedInteger()) |
| 202 | mtype = MPI_INT64_T; |
| 203 | else if (type.isInteger(width: 64)) |
| 204 | mtype = MPI_UINT64_T; |
| 205 | else if (type.isInteger(width: 32) && !type.isUnsignedInteger()) |
| 206 | mtype = MPI_INT32_T; |
| 207 | else if (type.isInteger(width: 32)) |
| 208 | mtype = MPI_UINT32_T; |
| 209 | else if (type.isInteger(width: 16) && !type.isUnsignedInteger()) |
| 210 | mtype = MPI_INT16_T; |
| 211 | else if (type.isInteger(width: 16)) |
| 212 | mtype = MPI_UINT16_T; |
| 213 | else if (type.isInteger(width: 8) && !type.isUnsignedInteger()) |
| 214 | mtype = MPI_INT8_T; |
| 215 | else if (type.isInteger(width: 8)) |
| 216 | mtype = MPI_UINT8_T; |
| 217 | else |
| 218 | assert(false && "unsupported type" ); |
| 219 | return LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: rewriter.getI32Type(), |
| 220 | value: mtype); |
| 221 | } |
| 222 | |
| 223 | Value getMPIOp(const Location loc, ConversionPatternRewriter &rewriter, |
| 224 | mpi::MPI_ReductionOpEnum opAttr) override { |
| 225 | int32_t op = MPI_NO_OP; |
| 226 | switch (opAttr) { |
| 227 | case mpi::MPI_ReductionOpEnum::MPI_OP_NULL: |
| 228 | op = MPI_NO_OP; |
| 229 | break; |
| 230 | case mpi::MPI_ReductionOpEnum::MPI_MAX: |
| 231 | op = MPI_MAX; |
| 232 | break; |
| 233 | case mpi::MPI_ReductionOpEnum::MPI_MIN: |
| 234 | op = MPI_MIN; |
| 235 | break; |
| 236 | case mpi::MPI_ReductionOpEnum::MPI_SUM: |
| 237 | op = MPI_SUM; |
| 238 | break; |
| 239 | case mpi::MPI_ReductionOpEnum::MPI_PROD: |
| 240 | op = MPI_PROD; |
| 241 | break; |
| 242 | case mpi::MPI_ReductionOpEnum::MPI_LAND: |
| 243 | op = MPI_LAND; |
| 244 | break; |
| 245 | case mpi::MPI_ReductionOpEnum::MPI_BAND: |
| 246 | op = MPI_BAND; |
| 247 | break; |
| 248 | case mpi::MPI_ReductionOpEnum::MPI_LOR: |
| 249 | op = MPI_LOR; |
| 250 | break; |
| 251 | case mpi::MPI_ReductionOpEnum::MPI_BOR: |
| 252 | op = MPI_BOR; |
| 253 | break; |
| 254 | case mpi::MPI_ReductionOpEnum::MPI_LXOR: |
| 255 | op = MPI_LXOR; |
| 256 | break; |
| 257 | case mpi::MPI_ReductionOpEnum::MPI_BXOR: |
| 258 | op = MPI_BXOR; |
| 259 | break; |
| 260 | case mpi::MPI_ReductionOpEnum::MPI_MINLOC: |
| 261 | op = MPI_MINLOC; |
| 262 | break; |
| 263 | case mpi::MPI_ReductionOpEnum::MPI_MAXLOC: |
| 264 | op = MPI_MAXLOC; |
| 265 | break; |
| 266 | case mpi::MPI_ReductionOpEnum::MPI_REPLACE: |
| 267 | op = MPI_REPLACE; |
| 268 | break; |
| 269 | } |
| 270 | return LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: rewriter.getI32Type(), value: op); |
| 271 | } |
| 272 | }; |
| 273 | |
| 274 | //===----------------------------------------------------------------------===// |
| 275 | // Implementation details for OpenMPI |
| 276 | //===----------------------------------------------------------------------===// |
| 277 | class OMPIImplTraits : public MPIImplTraits { |
| 278 | LLVM::GlobalOp getOrDefineExternalStruct(const Location loc, |
| 279 | ConversionPatternRewriter &rewriter, |
| 280 | StringRef name, |
| 281 | LLVM::LLVMStructType type) { |
| 282 | |
| 283 | return getOrDefineGlobal<LLVM::GlobalOp>( |
| 284 | moduleOp&: getModuleOp(), loc, rewriter, name, args&: type, /*isConstant=*/args: false, |
| 285 | args: LLVM::Linkage::External, args&: name, |
| 286 | /*value=*/args: Attribute(), /*alignment=*/args: 0, args: 0); |
| 287 | } |
| 288 | |
| 289 | public: |
| 290 | using MPIImplTraits::MPIImplTraits; |
| 291 | |
| 292 | ~OMPIImplTraits() override = default; |
| 293 | |
| 294 | Value getCommWorld(const Location loc, |
| 295 | ConversionPatternRewriter &rewriter) override { |
| 296 | auto *context = rewriter.getContext(); |
| 297 | // get external opaque struct pointer type |
| 298 | auto commStructT = |
| 299 | LLVM::LLVMStructType::getOpaque(name: "ompi_communicator_t" , context); |
| 300 | StringRef name = "ompi_mpi_comm_world" ; |
| 301 | |
| 302 | // make sure global op definition exists |
| 303 | getOrDefineExternalStruct(loc, rewriter, name, type: commStructT); |
| 304 | |
| 305 | // get address of symbol |
| 306 | auto comm = LLVM::AddressOfOp::create(builder&: rewriter, location: loc, |
| 307 | res: LLVM::LLVMPointerType::get(context), |
| 308 | global_name: SymbolRefAttr::get(ctx: context, value: name)); |
| 309 | return LLVM::PtrToIntOp::create(builder&: rewriter, location: loc, res: rewriter.getI64Type(), arg: comm); |
| 310 | } |
| 311 | |
| 312 | Value castComm(const Location loc, ConversionPatternRewriter &rewriter, |
| 313 | Value comm) override { |
| 314 | return LLVM::IntToPtrOp::create( |
| 315 | builder&: rewriter, location: loc, resultType: LLVM::LLVMPointerType::get(context: rewriter.getContext()), operands: comm); |
| 316 | } |
| 317 | |
| 318 | intptr_t getStatusIgnore() override { return 0; } |
| 319 | |
| 320 | void *getInPlace() override { return reinterpret_cast<void *>(1); } |
| 321 | |
| 322 | Value getDataType(const Location loc, ConversionPatternRewriter &rewriter, |
| 323 | Type type) override { |
| 324 | StringRef mtype; |
| 325 | if (type.isF32()) |
| 326 | mtype = "ompi_mpi_float" ; |
| 327 | else if (type.isF64()) |
| 328 | mtype = "ompi_mpi_double" ; |
| 329 | else if (type.isInteger(width: 64) && !type.isUnsignedInteger()) |
| 330 | mtype = "ompi_mpi_int64_t" ; |
| 331 | else if (type.isInteger(width: 64)) |
| 332 | mtype = "ompi_mpi_uint64_t" ; |
| 333 | else if (type.isInteger(width: 32) && !type.isUnsignedInteger()) |
| 334 | mtype = "ompi_mpi_int32_t" ; |
| 335 | else if (type.isInteger(width: 32)) |
| 336 | mtype = "ompi_mpi_uint32_t" ; |
| 337 | else if (type.isInteger(width: 16) && !type.isUnsignedInteger()) |
| 338 | mtype = "ompi_mpi_int16_t" ; |
| 339 | else if (type.isInteger(width: 16)) |
| 340 | mtype = "ompi_mpi_uint16_t" ; |
| 341 | else if (type.isInteger(width: 8) && !type.isUnsignedInteger()) |
| 342 | mtype = "ompi_mpi_int8_t" ; |
| 343 | else if (type.isInteger(width: 8)) |
| 344 | mtype = "ompi_mpi_uint8_t" ; |
| 345 | else |
| 346 | assert(false && "unsupported type" ); |
| 347 | |
| 348 | auto *context = rewriter.getContext(); |
| 349 | // get external opaque struct pointer type |
| 350 | auto typeStructT = |
| 351 | LLVM::LLVMStructType::getOpaque(name: "ompi_predefined_datatype_t" , context); |
| 352 | // make sure global op definition exists |
| 353 | getOrDefineExternalStruct(loc, rewriter, name: mtype, type: typeStructT); |
| 354 | // get address of symbol |
| 355 | return LLVM::AddressOfOp::create(builder&: rewriter, location: loc, |
| 356 | res: LLVM::LLVMPointerType::get(context), |
| 357 | global_name: SymbolRefAttr::get(ctx: context, value: mtype)); |
| 358 | } |
| 359 | |
| 360 | Value getMPIOp(const Location loc, ConversionPatternRewriter &rewriter, |
| 361 | mpi::MPI_ReductionOpEnum opAttr) override { |
| 362 | StringRef op; |
| 363 | switch (opAttr) { |
| 364 | case mpi::MPI_ReductionOpEnum::MPI_OP_NULL: |
| 365 | op = "ompi_mpi_no_op" ; |
| 366 | break; |
| 367 | case mpi::MPI_ReductionOpEnum::MPI_MAX: |
| 368 | op = "ompi_mpi_max" ; |
| 369 | break; |
| 370 | case mpi::MPI_ReductionOpEnum::MPI_MIN: |
| 371 | op = "ompi_mpi_min" ; |
| 372 | break; |
| 373 | case mpi::MPI_ReductionOpEnum::MPI_SUM: |
| 374 | op = "ompi_mpi_sum" ; |
| 375 | break; |
| 376 | case mpi::MPI_ReductionOpEnum::MPI_PROD: |
| 377 | op = "ompi_mpi_prod" ; |
| 378 | break; |
| 379 | case mpi::MPI_ReductionOpEnum::MPI_LAND: |
| 380 | op = "ompi_mpi_land" ; |
| 381 | break; |
| 382 | case mpi::MPI_ReductionOpEnum::MPI_BAND: |
| 383 | op = "ompi_mpi_band" ; |
| 384 | break; |
| 385 | case mpi::MPI_ReductionOpEnum::MPI_LOR: |
| 386 | op = "ompi_mpi_lor" ; |
| 387 | break; |
| 388 | case mpi::MPI_ReductionOpEnum::MPI_BOR: |
| 389 | op = "ompi_mpi_bor" ; |
| 390 | break; |
| 391 | case mpi::MPI_ReductionOpEnum::MPI_LXOR: |
| 392 | op = "ompi_mpi_lxor" ; |
| 393 | break; |
| 394 | case mpi::MPI_ReductionOpEnum::MPI_BXOR: |
| 395 | op = "ompi_mpi_bxor" ; |
| 396 | break; |
| 397 | case mpi::MPI_ReductionOpEnum::MPI_MINLOC: |
| 398 | op = "ompi_mpi_minloc" ; |
| 399 | break; |
| 400 | case mpi::MPI_ReductionOpEnum::MPI_MAXLOC: |
| 401 | op = "ompi_mpi_maxloc" ; |
| 402 | break; |
| 403 | case mpi::MPI_ReductionOpEnum::MPI_REPLACE: |
| 404 | op = "ompi_mpi_replace" ; |
| 405 | break; |
| 406 | } |
| 407 | auto *context = rewriter.getContext(); |
| 408 | // get external opaque struct pointer type |
| 409 | auto opStructT = |
| 410 | LLVM::LLVMStructType::getOpaque(name: "ompi_predefined_op_t" , context); |
| 411 | // make sure global op definition exists |
| 412 | getOrDefineExternalStruct(loc, rewriter, name: op, type: opStructT); |
| 413 | // get address of symbol |
| 414 | return LLVM::AddressOfOp::create(builder&: rewriter, location: loc, |
| 415 | res: LLVM::LLVMPointerType::get(context), |
| 416 | global_name: SymbolRefAttr::get(ctx: context, value: op)); |
| 417 | } |
| 418 | }; |
| 419 | |
| 420 | std::unique_ptr<MPIImplTraits> MPIImplTraits::get(ModuleOp &moduleOp) { |
| 421 | auto attr = dlti::query(op: moduleOp, keys: {"MPI:Implementation" }, emitError: false); |
| 422 | if (failed(Result: attr)) |
| 423 | return std::make_unique<MPICHImplTraits>(args&: moduleOp); |
| 424 | auto strAttr = dyn_cast<StringAttr>(Val&: attr.value()); |
| 425 | if (strAttr && strAttr.getValue() == "OpenMPI" ) |
| 426 | return std::make_unique<OMPIImplTraits>(args&: moduleOp); |
| 427 | if (!strAttr || strAttr.getValue() != "MPICH" ) |
| 428 | moduleOp.emitWarning() << "Unknown \"MPI:Implementation\" value in DLTI (" |
| 429 | << (strAttr ? strAttr.getValue() : "<NULL>" ) |
| 430 | << "), defaulting to MPICH" ; |
| 431 | return std::make_unique<MPICHImplTraits>(args&: moduleOp); |
| 432 | } |
| 433 | |
| 434 | //===----------------------------------------------------------------------===// |
| 435 | // InitOpLowering |
| 436 | //===----------------------------------------------------------------------===// |
| 437 | |
| 438 | struct InitOpLowering : public ConvertOpToLLVMPattern<mpi::InitOp> { |
| 439 | using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; |
| 440 | |
| 441 | LogicalResult |
| 442 | matchAndRewrite(mpi::InitOp op, OpAdaptor adaptor, |
| 443 | ConversionPatternRewriter &rewriter) const override { |
| 444 | Location loc = op.getLoc(); |
| 445 | |
| 446 | // ptrType `!llvm.ptr` |
| 447 | Type ptrType = LLVM::LLVMPointerType::get(context: rewriter.getContext()); |
| 448 | |
| 449 | // instantiate nullptr `%nullptr = llvm.mlir.zero : !llvm.ptr` |
| 450 | auto nullPtrOp = LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: ptrType); |
| 451 | Value llvmnull = nullPtrOp.getRes(); |
| 452 | |
| 453 | // grab a reference to the global module op: |
| 454 | auto moduleOp = op->getParentOfType<ModuleOp>(); |
| 455 | |
| 456 | // LLVM Function type representing `i32 MPI_Init(ptr, ptr)` |
| 457 | auto initFuncType = |
| 458 | LLVM::LLVMFunctionType::get(result: rewriter.getI32Type(), arguments: {ptrType, ptrType}); |
| 459 | // get or create function declaration: |
| 460 | LLVM::LLVMFuncOp initDecl = |
| 461 | getOrDefineFunction(moduleOp, loc, rewriter, name: "MPI_Init" , type: initFuncType); |
| 462 | |
| 463 | // replace init with function call |
| 464 | rewriter.replaceOpWithNewOp<LLVM::CallOp>(op, args&: initDecl, |
| 465 | args: ValueRange{llvmnull, llvmnull}); |
| 466 | |
| 467 | return success(); |
| 468 | } |
| 469 | }; |
| 470 | |
| 471 | //===----------------------------------------------------------------------===// |
| 472 | // FinalizeOpLowering |
| 473 | //===----------------------------------------------------------------------===// |
| 474 | |
| 475 | struct FinalizeOpLowering : public ConvertOpToLLVMPattern<mpi::FinalizeOp> { |
| 476 | using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; |
| 477 | |
| 478 | LogicalResult |
| 479 | matchAndRewrite(mpi::FinalizeOp op, OpAdaptor adaptor, |
| 480 | ConversionPatternRewriter &rewriter) const override { |
| 481 | // get loc |
| 482 | Location loc = op.getLoc(); |
| 483 | |
| 484 | // grab a reference to the global module op: |
| 485 | auto moduleOp = op->getParentOfType<ModuleOp>(); |
| 486 | |
| 487 | // LLVM Function type representing `i32 MPI_Finalize()` |
| 488 | auto initFuncType = LLVM::LLVMFunctionType::get(result: rewriter.getI32Type(), arguments: {}); |
| 489 | // get or create function declaration: |
| 490 | LLVM::LLVMFuncOp initDecl = getOrDefineFunction( |
| 491 | moduleOp, loc, rewriter, name: "MPI_Finalize" , type: initFuncType); |
| 492 | |
| 493 | // replace init with function call |
| 494 | rewriter.replaceOpWithNewOp<LLVM::CallOp>(op, args&: initDecl, args: ValueRange{}); |
| 495 | |
| 496 | return success(); |
| 497 | } |
| 498 | }; |
| 499 | |
| 500 | //===----------------------------------------------------------------------===// |
| 501 | // CommWorldOpLowering |
| 502 | //===----------------------------------------------------------------------===// |
| 503 | |
| 504 | struct CommWorldOpLowering : public ConvertOpToLLVMPattern<mpi::CommWorldOp> { |
| 505 | using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; |
| 506 | |
| 507 | LogicalResult |
| 508 | matchAndRewrite(mpi::CommWorldOp op, OpAdaptor adaptor, |
| 509 | ConversionPatternRewriter &rewriter) const override { |
| 510 | // grab a reference to the global module op: |
| 511 | auto moduleOp = op->getParentOfType<ModuleOp>(); |
| 512 | auto mpiTraits = MPIImplTraits::get(moduleOp); |
| 513 | // get MPI_COMM_WORLD |
| 514 | rewriter.replaceOp(op, newValues: mpiTraits->getCommWorld(loc: op.getLoc(), rewriter)); |
| 515 | |
| 516 | return success(); |
| 517 | } |
| 518 | }; |
| 519 | |
| 520 | //===----------------------------------------------------------------------===// |
| 521 | // CommSplitOpLowering |
| 522 | //===----------------------------------------------------------------------===// |
| 523 | |
| 524 | struct CommSplitOpLowering : public ConvertOpToLLVMPattern<mpi::CommSplitOp> { |
| 525 | using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; |
| 526 | |
| 527 | LogicalResult |
| 528 | matchAndRewrite(mpi::CommSplitOp op, OpAdaptor adaptor, |
| 529 | ConversionPatternRewriter &rewriter) const override { |
| 530 | // grab a reference to the global module op: |
| 531 | auto moduleOp = op->getParentOfType<ModuleOp>(); |
| 532 | auto mpiTraits = MPIImplTraits::get(moduleOp); |
| 533 | Type i32 = rewriter.getI32Type(); |
| 534 | Type ptrType = LLVM::LLVMPointerType::get(context: op->getContext()); |
| 535 | Location loc = op.getLoc(); |
| 536 | |
| 537 | // get communicator |
| 538 | Value comm = mpiTraits->castComm(loc, rewriter, comm: adaptor.getComm()); |
| 539 | auto one = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i32, value: 1); |
| 540 | auto outPtr = |
| 541 | LLVM::AllocaOp::create(builder&: rewriter, location: loc, resultType: ptrType, elementType: comm.getType(), arraySize: one); |
| 542 | |
| 543 | // int MPI_Comm_split(MPI_Comm comm, int color, int key, MPI_Comm * newcomm) |
| 544 | auto funcType = |
| 545 | LLVM::LLVMFunctionType::get(result: i32, arguments: {comm.getType(), i32, i32, ptrType}); |
| 546 | // get or create function declaration: |
| 547 | LLVM::LLVMFuncOp funcDecl = getOrDefineFunction(moduleOp, loc, rewriter, |
| 548 | name: "MPI_Comm_split" , type: funcType); |
| 549 | |
| 550 | auto callOp = |
| 551 | LLVM::CallOp::create(builder&: rewriter, location: loc, func: funcDecl, |
| 552 | args: ValueRange{comm, adaptor.getColor(), |
| 553 | adaptor.getKey(), outPtr.getRes()}); |
| 554 | |
| 555 | // load the communicator into a register |
| 556 | Value res = LLVM::LoadOp::create(builder&: rewriter, location: loc, type: i32, addr: outPtr.getResult()); |
| 557 | res = LLVM::SExtOp::create(builder&: rewriter, location: loc, res: rewriter.getI64Type(), arg: res); |
| 558 | |
| 559 | // if retval is checked, replace uses of retval with the results from the |
| 560 | // call op |
| 561 | SmallVector<Value> replacements; |
| 562 | if (op.getRetval()) |
| 563 | replacements.push_back(Elt: callOp.getResult()); |
| 564 | |
| 565 | // replace op |
| 566 | replacements.push_back(Elt: res); |
| 567 | rewriter.replaceOp(op, newValues: replacements); |
| 568 | |
| 569 | return success(); |
| 570 | } |
| 571 | }; |
| 572 | |
| 573 | //===----------------------------------------------------------------------===// |
| 574 | // CommRankOpLowering |
| 575 | //===----------------------------------------------------------------------===// |
| 576 | |
| 577 | struct CommRankOpLowering : public ConvertOpToLLVMPattern<mpi::CommRankOp> { |
| 578 | using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; |
| 579 | |
| 580 | LogicalResult |
| 581 | matchAndRewrite(mpi::CommRankOp op, OpAdaptor adaptor, |
| 582 | ConversionPatternRewriter &rewriter) const override { |
| 583 | // get some helper vars |
| 584 | Location loc = op.getLoc(); |
| 585 | MLIRContext *context = rewriter.getContext(); |
| 586 | Type i32 = rewriter.getI32Type(); |
| 587 | |
| 588 | // ptrType `!llvm.ptr` |
| 589 | Type ptrType = LLVM::LLVMPointerType::get(context); |
| 590 | |
| 591 | // grab a reference to the global module op: |
| 592 | auto moduleOp = op->getParentOfType<ModuleOp>(); |
| 593 | |
| 594 | auto mpiTraits = MPIImplTraits::get(moduleOp); |
| 595 | // get communicator |
| 596 | Value comm = mpiTraits->castComm(loc, rewriter, comm: adaptor.getComm()); |
| 597 | |
| 598 | // LLVM Function type representing `i32 MPI_Comm_rank(ptr, ptr)` |
| 599 | auto rankFuncType = |
| 600 | LLVM::LLVMFunctionType::get(result: i32, arguments: {comm.getType(), ptrType}); |
| 601 | // get or create function declaration: |
| 602 | LLVM::LLVMFuncOp initDecl = getOrDefineFunction( |
| 603 | moduleOp, loc, rewriter, name: "MPI_Comm_rank" , type: rankFuncType); |
| 604 | |
| 605 | // replace with function call |
| 606 | auto one = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i32, value: 1); |
| 607 | auto rankptr = LLVM::AllocaOp::create(builder&: rewriter, location: loc, resultType: ptrType, elementType: i32, arraySize: one); |
| 608 | auto callOp = LLVM::CallOp::create(builder&: rewriter, location: loc, func: initDecl, |
| 609 | args: ValueRange{comm, rankptr.getRes()}); |
| 610 | |
| 611 | // load the rank into a register |
| 612 | auto loadedRank = |
| 613 | LLVM::LoadOp::create(builder&: rewriter, location: loc, type: i32, addr: rankptr.getResult()); |
| 614 | |
| 615 | // if retval is checked, replace uses of retval with the results from the |
| 616 | // call op |
| 617 | SmallVector<Value> replacements; |
| 618 | if (op.getRetval()) |
| 619 | replacements.push_back(Elt: callOp.getResult()); |
| 620 | |
| 621 | // replace all uses, then erase op |
| 622 | replacements.push_back(Elt: loadedRank.getRes()); |
| 623 | rewriter.replaceOp(op, newValues: replacements); |
| 624 | |
| 625 | return success(); |
| 626 | } |
| 627 | }; |
| 628 | |
| 629 | //===----------------------------------------------------------------------===// |
| 630 | // CommSizeOpLowering |
| 631 | //===----------------------------------------------------------------------===// |
| 632 | |
| 633 | static Value createOrFoldCommSize(ConversionPatternRewriter &rewriter, |
| 634 | Location loc, Value commOrg, |
| 635 | Value commAdapt) { |
| 636 | auto i32 = rewriter.getI32Type(); |
| 637 | auto nRanksOp = mpi::CommSizeOp::create(builder&: rewriter, location: loc, resultTypes: i32, comm: commOrg); |
| 638 | if (succeeded(Result: FoldToDLTIConst(op: nRanksOp, key: "MPI:comm_world_size" , b&: rewriter))) |
| 639 | return nRanksOp.getSize(); |
| 640 | rewriter.eraseOp(op: nRanksOp); |
| 641 | return mpi::CommSizeOp::create(builder&: rewriter, location: loc, resultTypes: i32, comm: commAdapt).getSize(); |
| 642 | } |
| 643 | |
| 644 | struct CommSizeOpLowering : public ConvertOpToLLVMPattern<mpi::CommSizeOp> { |
| 645 | using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; |
| 646 | |
| 647 | LogicalResult |
| 648 | matchAndRewrite(mpi::CommSizeOp op, OpAdaptor adaptor, |
| 649 | ConversionPatternRewriter &rewriter) const override { |
| 650 | // get some helper vars |
| 651 | Location loc = op.getLoc(); |
| 652 | MLIRContext *context = rewriter.getContext(); |
| 653 | Type i32 = rewriter.getI32Type(); |
| 654 | |
| 655 | // ptrType `!llvm.ptr` |
| 656 | Type ptrType = LLVM::LLVMPointerType::get(context); |
| 657 | |
| 658 | // grab a reference to the global module op: |
| 659 | auto moduleOp = op->getParentOfType<ModuleOp>(); |
| 660 | |
| 661 | auto mpiTraits = MPIImplTraits::get(moduleOp); |
| 662 | // get communicator |
| 663 | Value comm = mpiTraits->castComm(loc, rewriter, comm: adaptor.getComm()); |
| 664 | |
| 665 | // LLVM Function type representing `i32 MPI_Comm_rank(ptr, ptr)` |
| 666 | auto SizeFuncType = |
| 667 | LLVM::LLVMFunctionType::get(result: i32, arguments: {comm.getType(), ptrType}); |
| 668 | // get or create function declaration: |
| 669 | LLVM::LLVMFuncOp initDecl = getOrDefineFunction( |
| 670 | moduleOp, loc, rewriter, name: "MPI_Comm_size" , type: SizeFuncType); |
| 671 | |
| 672 | // replace with function call |
| 673 | auto one = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i32, value: 1); |
| 674 | auto sizeptr = LLVM::AllocaOp::create(builder&: rewriter, location: loc, resultType: ptrType, elementType: i32, arraySize: one); |
| 675 | auto callOp = LLVM::CallOp::create(builder&: rewriter, location: loc, func: initDecl, |
| 676 | args: ValueRange{comm, sizeptr.getRes()}); |
| 677 | |
| 678 | // load the Size into a register |
| 679 | auto loadedSize = |
| 680 | LLVM::LoadOp::create(builder&: rewriter, location: loc, type: i32, addr: sizeptr.getResult()); |
| 681 | |
| 682 | // if retval is checked, replace uses of retval with the results from the |
| 683 | // call op |
| 684 | SmallVector<Value> replacements; |
| 685 | if (op.getRetval()) |
| 686 | replacements.push_back(Elt: callOp.getResult()); |
| 687 | |
| 688 | // replace all uses, then erase op |
| 689 | replacements.push_back(Elt: loadedSize.getRes()); |
| 690 | rewriter.replaceOp(op, newValues: replacements); |
| 691 | |
| 692 | return success(); |
| 693 | } |
| 694 | }; |
| 695 | |
| 696 | //===----------------------------------------------------------------------===// |
| 697 | // SendOpLowering |
| 698 | //===----------------------------------------------------------------------===// |
| 699 | |
| 700 | struct SendOpLowering : public ConvertOpToLLVMPattern<mpi::SendOp> { |
| 701 | using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; |
| 702 | |
| 703 | LogicalResult |
| 704 | matchAndRewrite(mpi::SendOp op, OpAdaptor adaptor, |
| 705 | ConversionPatternRewriter &rewriter) const override { |
| 706 | // get some helper vars |
| 707 | Location loc = op.getLoc(); |
| 708 | MLIRContext *context = rewriter.getContext(); |
| 709 | Type i32 = rewriter.getI32Type(); |
| 710 | Type elemType = op.getRef().getType().getElementType(); |
| 711 | int64_t rank = op.getRef().getType().getRank(); |
| 712 | |
| 713 | // ptrType `!llvm.ptr` |
| 714 | Type ptrType = LLVM::LLVMPointerType::get(context); |
| 715 | |
| 716 | // grab a reference to the global module op: |
| 717 | auto moduleOp = op->getParentOfType<ModuleOp>(); |
| 718 | |
| 719 | // get MPI_COMM_WORLD, dataType and pointer |
| 720 | auto [dataPtr, size] = |
| 721 | getRawPtrAndSize(loc, rewriter, memRef: adaptor.getRef(), rank, elType: elemType); |
| 722 | auto mpiTraits = MPIImplTraits::get(moduleOp); |
| 723 | Value dataType = mpiTraits->getDataType(loc, rewriter, type: elemType); |
| 724 | Value comm = mpiTraits->castComm(loc, rewriter, comm: adaptor.getComm()); |
| 725 | |
| 726 | // LLVM Function type representing `i32 MPI_send(data, count, datatype, dst, |
| 727 | // tag, comm)` |
| 728 | auto funcType = LLVM::LLVMFunctionType::get( |
| 729 | result: i32, arguments: {ptrType, i32, dataType.getType(), i32, i32, comm.getType()}); |
| 730 | // get or create function declaration: |
| 731 | LLVM::LLVMFuncOp funcDecl = |
| 732 | getOrDefineFunction(moduleOp, loc, rewriter, name: "MPI_Send" , type: funcType); |
| 733 | |
| 734 | // replace op with function call |
| 735 | auto funcCall = LLVM::CallOp::create(builder&: rewriter, location: loc, func: funcDecl, |
| 736 | args: ValueRange{dataPtr, size, dataType, |
| 737 | adaptor.getDest(), |
| 738 | adaptor.getTag(), comm}); |
| 739 | if (op.getRetval()) |
| 740 | rewriter.replaceOp(op, newValues: funcCall.getResult()); |
| 741 | else |
| 742 | rewriter.eraseOp(op); |
| 743 | |
| 744 | return success(); |
| 745 | } |
| 746 | }; |
| 747 | |
| 748 | //===----------------------------------------------------------------------===// |
| 749 | // RecvOpLowering |
| 750 | //===----------------------------------------------------------------------===// |
| 751 | |
| 752 | struct RecvOpLowering : public ConvertOpToLLVMPattern<mpi::RecvOp> { |
| 753 | using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; |
| 754 | |
| 755 | LogicalResult |
| 756 | matchAndRewrite(mpi::RecvOp op, OpAdaptor adaptor, |
| 757 | ConversionPatternRewriter &rewriter) const override { |
| 758 | // get some helper vars |
| 759 | Location loc = op.getLoc(); |
| 760 | MLIRContext *context = rewriter.getContext(); |
| 761 | Type i32 = rewriter.getI32Type(); |
| 762 | Type i64 = rewriter.getI64Type(); |
| 763 | Type elemType = op.getRef().getType().getElementType(); |
| 764 | int64_t rank = op.getRef().getType().getRank(); |
| 765 | |
| 766 | // ptrType `!llvm.ptr` |
| 767 | Type ptrType = LLVM::LLVMPointerType::get(context); |
| 768 | |
| 769 | // grab a reference to the global module op: |
| 770 | auto moduleOp = op->getParentOfType<ModuleOp>(); |
| 771 | |
| 772 | // get MPI_COMM_WORLD, dataType, status_ignore and pointer |
| 773 | auto [dataPtr, size] = |
| 774 | getRawPtrAndSize(loc, rewriter, memRef: adaptor.getRef(), rank, elType: elemType); |
| 775 | auto mpiTraits = MPIImplTraits::get(moduleOp); |
| 776 | Value dataType = mpiTraits->getDataType(loc, rewriter, type: elemType); |
| 777 | Value comm = mpiTraits->castComm(loc, rewriter, comm: adaptor.getComm()); |
| 778 | Value statusIgnore = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i64, |
| 779 | value: mpiTraits->getStatusIgnore()); |
| 780 | statusIgnore = |
| 781 | LLVM::IntToPtrOp::create(builder&: rewriter, location: loc, resultType: ptrType, operands: statusIgnore); |
| 782 | |
| 783 | // LLVM Function type representing `i32 MPI_Recv(data, count, datatype, dst, |
| 784 | // tag, comm)` |
| 785 | auto funcType = |
| 786 | LLVM::LLVMFunctionType::get(result: i32, arguments: {ptrType, i32, dataType.getType(), i32, |
| 787 | i32, comm.getType(), ptrType}); |
| 788 | // get or create function declaration: |
| 789 | LLVM::LLVMFuncOp funcDecl = |
| 790 | getOrDefineFunction(moduleOp, loc, rewriter, name: "MPI_Recv" , type: funcType); |
| 791 | |
| 792 | // replace op with function call |
| 793 | auto funcCall = LLVM::CallOp::create( |
| 794 | builder&: rewriter, location: loc, func: funcDecl, |
| 795 | args: ValueRange{dataPtr, size, dataType, adaptor.getSource(), |
| 796 | adaptor.getTag(), comm, statusIgnore}); |
| 797 | if (op.getRetval()) |
| 798 | rewriter.replaceOp(op, newValues: funcCall.getResult()); |
| 799 | else |
| 800 | rewriter.eraseOp(op); |
| 801 | |
| 802 | return success(); |
| 803 | } |
| 804 | }; |
| 805 | |
| 806 | //===----------------------------------------------------------------------===// |
| 807 | // AllGatherOpLowering |
| 808 | //===----------------------------------------------------------------------===// |
| 809 | |
| 810 | struct AllGatherOpLowering : public ConvertOpToLLVMPattern<mpi::AllGatherOp> { |
| 811 | using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; |
| 812 | |
| 813 | LogicalResult |
| 814 | matchAndRewrite(mpi::AllGatherOp op, OpAdaptor adaptor, |
| 815 | ConversionPatternRewriter &rewriter) const override { |
| 816 | Location loc = op.getLoc(); |
| 817 | MLIRContext *context = rewriter.getContext(); |
| 818 | Type sElemType = op.getSendbuf().getType().getElementType(); |
| 819 | Type rElemType = op.getRecvbuf().getType().getElementType(); |
| 820 | int64_t sRank = op.getSendbuf().getType().getRank(); |
| 821 | int64_t rRank = op.getRecvbuf().getType().getRank(); |
| 822 | auto [sendPtr, sendSize] = |
| 823 | getRawPtrAndSize(loc, rewriter, memRef: adaptor.getSendbuf(), rank: sRank, elType: sElemType); |
| 824 | auto [recvPtr, recvSize] = |
| 825 | getRawPtrAndSize(loc, rewriter, memRef: adaptor.getRecvbuf(), rank: rRank, elType: rElemType); |
| 826 | |
| 827 | auto moduleOp = op->getParentOfType<ModuleOp>(); |
| 828 | auto mpiTraits = MPIImplTraits::get(moduleOp); |
| 829 | Value sDataType = mpiTraits->getDataType(loc, rewriter, type: sElemType); |
| 830 | Value rDataType = mpiTraits->getDataType(loc, rewriter, type: rElemType); |
| 831 | Value comm = mpiTraits->castComm(loc, rewriter, comm: adaptor.getComm()); |
| 832 | |
| 833 | Type ptrType = LLVM::LLVMPointerType::get(context); |
| 834 | Type i32 = rewriter.getI32Type(); |
| 835 | // int MPI_Allgather( |
| 836 | // const void* buffer_send, int count_send, MPI_Datatype datatype_send, |
| 837 | // void* buffer_recv, int count_recv, MPI_Datatype datatype_recv, |
| 838 | // MPI_Comm communicator); |
| 839 | auto funcType = LLVM::LLVMFunctionType::get( |
| 840 | result: i32, arguments: {ptrType, i32, sDataType.getType(), ptrType, i32, |
| 841 | rDataType.getType(), comm.getType()}); |
| 842 | // get or create function declaration: |
| 843 | LLVM::LLVMFuncOp funcDecl = |
| 844 | getOrDefineFunction(moduleOp, loc, rewriter, name: "MPI_Allgather" , type: funcType); |
| 845 | |
| 846 | // count_recv is the number of elements received from each rank, not total |
| 847 | Value nRanks = |
| 848 | createOrFoldCommSize(rewriter, loc, commOrg: op.getComm(), commAdapt: adaptor.getComm()); |
| 849 | Value recvCountPerRank = |
| 850 | LLVM::UDivOp::create(builder&: rewriter, location: loc, res: i32, lhs: recvSize, rhs: nRanks); |
| 851 | |
| 852 | // replace op with function call |
| 853 | auto funcCall = |
| 854 | LLVM::CallOp::create(builder&: rewriter, location: loc, func: funcDecl, |
| 855 | args: ValueRange{sendPtr, sendSize, sDataType, recvPtr, |
| 856 | recvCountPerRank, rDataType, comm}); |
| 857 | |
| 858 | if (op.getRetval()) |
| 859 | rewriter.replaceOp(op, newValues: funcCall.getResult()); |
| 860 | else |
| 861 | rewriter.eraseOp(op); |
| 862 | |
| 863 | return success(); |
| 864 | } |
| 865 | }; |
| 866 | |
| 867 | //===----------------------------------------------------------------------===// |
| 868 | // AllReduceOpLowering |
| 869 | //===----------------------------------------------------------------------===// |
| 870 | |
| 871 | struct AllReduceOpLowering : public ConvertOpToLLVMPattern<mpi::AllReduceOp> { |
| 872 | using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; |
| 873 | |
| 874 | LogicalResult |
| 875 | matchAndRewrite(mpi::AllReduceOp op, OpAdaptor adaptor, |
| 876 | ConversionPatternRewriter &rewriter) const override { |
| 877 | Location loc = op.getLoc(); |
| 878 | MLIRContext *context = rewriter.getContext(); |
| 879 | Type i32 = rewriter.getI32Type(); |
| 880 | Type i64 = rewriter.getI64Type(); |
| 881 | Type elemType = op.getSendbuf().getType().getElementType(); |
| 882 | int64_t sRank = op.getSendbuf().getType().getRank(); |
| 883 | int64_t rRank = op.getRecvbuf().getType().getRank(); |
| 884 | |
| 885 | // ptrType `!llvm.ptr` |
| 886 | Type ptrType = LLVM::LLVMPointerType::get(context); |
| 887 | auto moduleOp = op->getParentOfType<ModuleOp>(); |
| 888 | auto mpiTraits = MPIImplTraits::get(moduleOp); |
| 889 | auto [sendPtr, sendSize] = |
| 890 | getRawPtrAndSize(loc, rewriter, memRef: adaptor.getSendbuf(), rank: sRank, elType: elemType); |
| 891 | auto [recvPtr, recvSize] = |
| 892 | getRawPtrAndSize(loc, rewriter, memRef: adaptor.getRecvbuf(), rank: rRank, elType: elemType); |
| 893 | |
| 894 | // If input and output are the same, request in-place operation. |
| 895 | if (adaptor.getSendbuf() == adaptor.getRecvbuf()) { |
| 896 | sendPtr = LLVM::ConstantOp::create( |
| 897 | builder&: rewriter, location: loc, type: i64, |
| 898 | value: reinterpret_cast<int64_t>(mpiTraits->getInPlace())); |
| 899 | sendPtr = LLVM::IntToPtrOp::create(builder&: rewriter, location: loc, resultType: ptrType, operands: sendPtr); |
| 900 | } |
| 901 | |
| 902 | Value dataType = mpiTraits->getDataType(loc, rewriter, type: elemType); |
| 903 | Value mpiOp = mpiTraits->getMPIOp(loc, rewriter, opAttr: op.getOp()); |
| 904 | Value commWorld = mpiTraits->castComm(loc, rewriter, comm: adaptor.getComm()); |
| 905 | |
| 906 | // 'int MPI_Allreduce(const void *sendbuf, void *recvbuf, int count, |
| 907 | // MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)' |
| 908 | auto funcType = LLVM::LLVMFunctionType::get( |
| 909 | result: i32, arguments: {ptrType, ptrType, i32, dataType.getType(), mpiOp.getType(), |
| 910 | commWorld.getType()}); |
| 911 | // get or create function declaration: |
| 912 | LLVM::LLVMFuncOp funcDecl = |
| 913 | getOrDefineFunction(moduleOp, loc, rewriter, name: "MPI_Allreduce" , type: funcType); |
| 914 | |
| 915 | // replace op with function call |
| 916 | auto funcCall = LLVM::CallOp::create( |
| 917 | builder&: rewriter, location: loc, func: funcDecl, |
| 918 | args: ValueRange{sendPtr, recvPtr, sendSize, dataType, mpiOp, commWorld}); |
| 919 | |
| 920 | if (op.getRetval()) |
| 921 | rewriter.replaceOp(op, newValues: funcCall.getResult()); |
| 922 | else |
| 923 | rewriter.eraseOp(op); |
| 924 | |
| 925 | return success(); |
| 926 | } |
| 927 | }; |
| 928 | |
| 929 | //===----------------------------------------------------------------------===// |
| 930 | // ReduceScatterBlockOpLowering |
| 931 | //===----------------------------------------------------------------------===// |
| 932 | |
| 933 | struct ReduceScatterBlockOpLowering |
| 934 | : public ConvertOpToLLVMPattern<mpi::ReduceScatterBlockOp> { |
| 935 | using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; |
| 936 | |
| 937 | LogicalResult |
| 938 | matchAndRewrite(mpi::ReduceScatterBlockOp op, OpAdaptor adaptor, |
| 939 | ConversionPatternRewriter &rewriter) const override { |
| 940 | Location loc = op.getLoc(); |
| 941 | MLIRContext *context = rewriter.getContext(); |
| 942 | Type i32 = rewriter.getI32Type(); |
| 943 | Type i64 = rewriter.getI64Type(); |
| 944 | Type elemType = op.getSendbuf().getType().getElementType(); |
| 945 | int64_t sRank = op.getSendbuf().getType().getRank(); |
| 946 | int64_t rRank = op.getRecvbuf().getType().getRank(); |
| 947 | |
| 948 | // ptrType `!llvm.ptr` |
| 949 | Type ptrType = LLVM::LLVMPointerType::get(context); |
| 950 | auto moduleOp = op->getParentOfType<ModuleOp>(); |
| 951 | auto mpiTraits = MPIImplTraits::get(moduleOp); |
| 952 | auto [sendPtr, sendSize] = |
| 953 | getRawPtrAndSize(loc, rewriter, memRef: adaptor.getSendbuf(), rank: sRank, elType: elemType); |
| 954 | auto [recvPtr, recvSize] = |
| 955 | getRawPtrAndSize(loc, rewriter, memRef: adaptor.getRecvbuf(), rank: rRank, elType: elemType); |
| 956 | |
| 957 | // If input and output are the same, request in-place operation. |
| 958 | if (adaptor.getSendbuf() == adaptor.getRecvbuf()) { |
| 959 | sendPtr = LLVM::ConstantOp::create( |
| 960 | builder&: rewriter, location: loc, type: i64, |
| 961 | value: reinterpret_cast<int64_t>(mpiTraits->getInPlace())); |
| 962 | sendPtr = LLVM::IntToPtrOp::create(builder&: rewriter, location: loc, resultType: ptrType, operands: sendPtr); |
| 963 | } |
| 964 | |
| 965 | Value dataType = mpiTraits->getDataType(loc, rewriter, type: elemType); |
| 966 | Value mpiOp = mpiTraits->getMPIOp(loc, rewriter, opAttr: op.getOp()); |
| 967 | Value comm = mpiTraits->castComm(loc, rewriter, comm: adaptor.getComm()); |
| 968 | |
| 969 | Value nRanks = |
| 970 | createOrFoldCommSize(rewriter, loc, commOrg: op.getComm(), commAdapt: adaptor.getComm()); |
| 971 | Value totalExpected = |
| 972 | LLVM::MulOp::create(builder&: rewriter, location: loc, res: i32, lhs: recvSize, rhs: nRanks); |
| 973 | Value sizeIsValid = LLVM::ICmpOp::create( |
| 974 | builder&: rewriter, location: loc, predicate: LLVM::ICmpPredicate::eq, lhs: sendSize, rhs: totalExpected); |
| 975 | cf::AssertOp::create(builder&: rewriter, location: loc, arg: sizeIsValid, |
| 976 | msg: "Send buffer's size must be the receive buffer's size " |
| 977 | "times the number of ranks" ); |
| 978 | |
| 979 | // 'int MPI_Reduce_scatter_block(const void *sendbuf, void *recvbuf, |
| 980 | // int recvcount, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)' |
| 981 | auto funcType = LLVM::LLVMFunctionType::get( |
| 982 | result: i32, arguments: {ptrType, ptrType, i32, dataType.getType(), mpiOp.getType(), |
| 983 | comm.getType()}); |
| 984 | // get or create function declaration: |
| 985 | LLVM::LLVMFuncOp funcDecl = getOrDefineFunction( |
| 986 | moduleOp, loc, rewriter, name: "MPI_Reduce_scatter_block" , type: funcType); |
| 987 | |
| 988 | // replace op with function call |
| 989 | auto funcCall = LLVM::CallOp::create( |
| 990 | builder&: rewriter, location: loc, func: funcDecl, |
| 991 | args: ValueRange{sendPtr, recvPtr, recvSize, dataType, mpiOp, comm}); |
| 992 | |
| 993 | if (op.getRetval()) |
| 994 | rewriter.replaceOp(op, newValues: funcCall.getResult()); |
| 995 | else |
| 996 | rewriter.eraseOp(op); |
| 997 | |
| 998 | return success(); |
| 999 | } |
| 1000 | }; |
| 1001 | |
| 1002 | //===----------------------------------------------------------------------===// |
| 1003 | // ConvertToLLVMPatternInterface implementation |
| 1004 | //===----------------------------------------------------------------------===// |
| 1005 | |
| 1006 | /// Implement the interface to convert Func to LLVM. |
| 1007 | struct FuncToLLVMDialectInterface : public ConvertToLLVMPatternInterface { |
| 1008 | FuncToLLVMDialectInterface(Dialect *dialect) |
| 1009 | : ConvertToLLVMPatternInterface(dialect) {} |
| 1010 | |
| 1011 | /// Hook for derived dialect interface to provide conversion patterns |
| 1012 | /// and mark dialect legal for the conversion target. |
| 1013 | void populateConvertToLLVMConversionPatterns( |
| 1014 | ConversionTarget &target, LLVMTypeConverter &typeConverter, |
| 1015 | RewritePatternSet &patterns) const final { |
| 1016 | mpi::populateMPIToLLVMConversionPatterns(converter&: typeConverter, patterns); |
| 1017 | } |
| 1018 | }; |
| 1019 | } // namespace |
| 1020 | |
| 1021 | //===----------------------------------------------------------------------===// |
| 1022 | // Pattern Population |
| 1023 | //===----------------------------------------------------------------------===// |
| 1024 | |
| 1025 | void mpi::populateMPIToLLVMConversionPatterns(LLVMTypeConverter &converter, |
| 1026 | RewritePatternSet &patterns) { |
| 1027 | // Using i64 as a portable, intermediate type for !mpi.comm. |
| 1028 | // It would be nicer to somehow get the right type directly, but TLDI is not |
| 1029 | // available here. |
| 1030 | converter.addConversion(callback: [](mpi::CommType type) { |
| 1031 | return IntegerType::get(context: type.getContext(), width: 64); |
| 1032 | }); |
| 1033 | patterns.add<CommRankOpLowering, CommSizeOpLowering, CommSplitOpLowering, |
| 1034 | CommWorldOpLowering, FinalizeOpLowering, InitOpLowering, |
| 1035 | SendOpLowering, RecvOpLowering, AllGatherOpLowering, |
| 1036 | AllReduceOpLowering, ReduceScatterBlockOpLowering>(arg&: converter); |
| 1037 | } |
| 1038 | |
| 1039 | void mpi::registerConvertMPIToLLVMInterface(DialectRegistry ®istry) { |
| 1040 | registry.addExtension(extensionFn: +[](MLIRContext *ctx, mpi::MPIDialect *dialect) { |
| 1041 | dialect->addInterfaces<FuncToLLVMDialectInterface>(); |
| 1042 | }); |
| 1043 | } |
| 1044 | |