1//===- ACCToLLVMUtils.cpp - OpenACC to LLVM helpers -------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Conversion/OpenACCToLLVM/ACCToLLVMUtils.h"
10
11#include "mlir/Dialect/Arith/IR/Arith.h"
12#include "mlir/Dialect/LLVMIR/LLVMAggregateOps.h"
13#include "mlir/Dialect/LLVMIR/LLVMCallOps.h"
14#include "mlir/Dialect/LLVMIR/LLVMConstantOps.h"
15#include "mlir/Dialect/LLVMIR/LLVMMemoryOps.h"
16#include "mlir/Dialect/LLVMIR/LLVMModuleOps.h"
17#include "mlir/Dialect/LLVMIR/LLVMTerminatorOps.h"
18#include "mlir/Interfaces/FunctionInterfaces.h"
19#include "llvm/ADT/Hashing.h"
20#include "llvm/ADT/STLExtras.h"
21#include "llvm/ADT/SmallString.h"
22#include "llvm/ADT/StringExtras.h"
23#include "llvm/ADT/Twine.h"
24
25#include <iterator>
26
27using namespace mlir;
28using namespace mlir::acc;
29
30Location acc::unfuseLoc(Location loc) {
31 while (auto fusedLoc = dyn_cast<FusedLoc>(Val&: loc))
32 loc = fusedLoc.getLocations().back();
33 return loc;
34}
35
36std::optional<FileLineColLoc>
37acc::getFileLineColLoc(Location loc, bool errorOnInvalidLocation) {
38 Location unfusedLoc = unfuseLoc(loc);
39
40 if (auto fileLoc = dyn_cast<FileLineColLoc>(Val&: unfusedLoc))
41 return fileLoc;
42
43 if (auto callSiteLoc = dyn_cast<CallSiteLoc>(Val&: unfusedLoc)) {
44 if (auto calleeFileLoc = getFileLineColLoc(loc: callSiteLoc.getCallee(), errorOnInvalidLocation: false))
45 return calleeFileLoc;
46 if (auto callerFileLoc =
47 getFileLineColLoc(loc: callSiteLoc.getCaller(), errorOnInvalidLocation))
48 return callerFileLoc;
49 }
50
51 if (errorOnInvalidLocation)
52 llvm_unreachable(
53 "cannot get file:line information: invalid Location information");
54 return std::nullopt;
55}
56
57StringRef acc::getParentFunctionName(Operation *op) {
58 if (!op)
59 return "";
60 auto funcOp = dyn_cast<FunctionOpInterface>(Val: op);
61 if (!funcOp)
62 funcOp = op->getParentOfType<FunctionOpInterface>();
63 return funcOp ? funcOp.getName() : StringRef();
64}
65
66StringRef acc::getParentFunctionName(Value value) {
67 if (auto *op = value.getDefiningOp())
68 return getParentFunctionName(op);
69 return "";
70}
71
72StringRef acc::getParentFunctionName(ValueRange values) {
73 for (Value value : values) {
74 if (StringRef name = getParentFunctionName(value); !name.empty())
75 return name;
76 }
77 return "";
78}
79
80std::string acc::getInternalGlobalName(StringRef kind, StringRef detail) {
81 SmallString<64> name("acc.");
82 name += kind;
83 if (detail.empty())
84 return name.str().str();
85
86 name += '.';
87 for (unsigned char c : detail)
88 name += (llvm::isAlnum(C: c) || c == '_' || c == '$' || c == '.') ? c : '_';
89 return name.str().str();
90}
91
92/// Creates or reuses a module-internal null-terminated string global and
93/// returns the GlobalOp.
94static LLVM::GlobalOp getOrCreateGlobalStringOp(Location loc,
95 OpBuilder &builder,
96 StringRef name, StringRef value,
97 Region &globalSymbolRegion,
98 SymbolTable *symbolTable) {
99 SmallString<32> nullTermStr(value);
100 nullTermStr.push_back(Elt: '\0');
101 StringAttr valueAttr = builder.getStringAttr(bytes: nullTermStr);
102
103 // A name says what the global holds, so a global of that name is the one to
104 // reuse as long as it holds this very value. Names come out of detail the
105 // source spells and are not one to one with it, so a name can also lead to a
106 // global holding something else, which a suffix then keeps apart.
107 SmallString<64> uniqueName(name);
108 if (symbolTable) {
109 unsigned suffix = 0;
110 while (Operation *taken = symbolTable->lookup(name: uniqueName)) {
111 auto global = dyn_cast<LLVM::GlobalOp>(Val: taken);
112 if (global && global.getValueOrNull() == valueAttr)
113 return global;
114 uniqueName.clear();
115 (name + "." + Twine(suffix++)).toVector(Out&: uniqueName);
116 }
117 }
118
119 // Materialize the global through the incoming builder so that it stays
120 // tracked when the caller is a dialect conversion rewriter.
121 OpBuilder::InsertionGuard guard(builder);
122 builder.setInsertionPointToStart(&globalSymbolRegion.front());
123 auto arrayTy = LLVM::LLVMArrayType::get(elementType: builder.getI8Type(),
124 numElements: nullTermStr.size_in_bytes());
125 auto global =
126 LLVM::GlobalOp::create(builder, location: loc, type: arrayTy, /*isConstant=*/true,
127 linkage: LLVM::Linkage::Internal, name: uniqueName, value: valueAttr,
128 /*alignment=*/0);
129 if (symbolTable)
130 symbolTable->insert(symbol: global);
131 return global;
132}
133
134Value acc::getOrCreateGlobalString(Location loc, OpBuilder &builder,
135 StringRef name, StringRef value,
136 Region &globalSymbolRegion,
137 SymbolTable *symbolTable) {
138 Type i64Ty = builder.getI64Type();
139 Type ptrTy = LLVM::LLVMPointerType::get(context: builder.getContext());
140 LLVM::GlobalOp global = getOrCreateGlobalStringOp(
141 loc, builder, name, value, globalSymbolRegion, symbolTable);
142
143 Value globalPtr = LLVM::AddressOfOp::create(builder, location: loc, global);
144 Value cst0 = LLVM::ConstantOp::create(builder, location: loc, res: i64Ty,
145 value: builder.getI64IntegerAttr(value: 0));
146 return LLVM::GEPOp::create(builder, location: loc, resultType: ptrTy, elementType: global.getType(), basePtr: globalPtr,
147 indices: ArrayRef<Value>({cst0, cst0}));
148}
149
150Value acc::createIdent(Location loc, StringRef functionName, OpBuilder &builder,
151 Region &globalSymbolRegion,
152 const ACCRuntimeCallConfig &config,
153 SymbolTable *symbolTable) {
154 MLIRContext *ctx = builder.getContext();
155 Type i32Ty = builder.getI32Type();
156 Type i64Ty = builder.getI64Type();
157 Type ptrTy = LLVM::LLVMPointerType::get(context: ctx);
158 Type structTy = LLVM::LLVMStructType::getLiteral(
159 context: ctx, types: {i32Ty, i32Ty, i32Ty, i32Ty, ptrTy});
160
161 std::string source;
162 // The position the globals describe is what names them, so a global found
163 // under such a name holds this very source string or ident.
164 std::string position;
165 if (auto fileLineColLoc =
166 getFileLineColLoc(loc, /*errorOnInvalidLocation=*/false)) {
167 std::string filename = fileLineColLoc->getFilename().str();
168 std::string line = std::to_string(val: fileLineColLoc->getLine());
169 std::string column = std::to_string(val: fileLineColLoc->getColumn());
170 std::string functionDisplayName =
171 functionName.empty() ? std::string()
172 : config.getFunctionDisplayName(mangledOrSymbol: functionName);
173 source = ";";
174 source += filename + ";";
175 source += functionDisplayName + ";";
176 source += line + ";";
177 source += column + ";";
178 source += ";";
179 position = line + "." + column + ".";
180 position += std::to_string(val: static_cast<uint64_t>(llvm::hash_value(arg: source)));
181 } else {
182 source = ";unknown;unknown;0;0;;";
183 position = "unknown";
184 }
185
186 std::string identGlobalName = getInternalGlobalName(kind: "ident", detail: position);
187 LLVM::GlobalOp identGlobal =
188 symbolTable ? symbolTable->lookup<LLVM::GlobalOp>(name: identGlobalName)
189 : LLVM::GlobalOp();
190 if (!identGlobal) {
191 LLVM::GlobalOp sourceGlobal = getOrCreateGlobalStringOp(
192 loc, builder, name: getInternalGlobalName(kind: "loc", detail: position), value: source,
193 globalSymbolRegion, symbolTable);
194
195 OpBuilder::InsertionGuard guard(builder);
196 builder.setInsertionPointAfter(sourceGlobal);
197 identGlobal = LLVM::GlobalOp::create(
198 builder, location: loc, type: structTy, /*isConstant=*/true, linkage: LLVM::Linkage::Internal,
199 name: identGlobalName, /*value=*/Attribute(), /*alignment=*/0);
200 if (symbolTable)
201 symbolTable->insert(symbol: identGlobal);
202
203 Block *block = builder.createBlock(parent: &identGlobal.getInitializerRegion());
204 builder.setInsertionPointToStart(block);
205 Value ident = LLVM::ZeroOp::create(builder, location: loc, res: structTy);
206 Value sourceBase = LLVM::AddressOfOp::create(builder, location: loc, global: sourceGlobal);
207 Value cst0 = LLVM::ConstantOp::create(builder, location: loc, res: i64Ty,
208 value: builder.getI64IntegerAttr(value: 0));
209 Value sourcePtr =
210 LLVM::GEPOp::create(builder, location: loc, resultType: ptrTy, elementType: sourceGlobal.getType(),
211 basePtr: sourceBase, indices: ArrayRef<Value>({cst0, cst0}));
212 ident = LLVM::InsertValueOp::create(builder, location: loc, res: structTy, container: ident,
213 value: sourcePtr, position: ArrayRef<int64_t>{4});
214 LLVM::ReturnOp::create(builder, location: loc, arg: ident);
215 }
216
217 return LLVM::AddressOfOp::create(builder, location: loc, global: identGlobal);
218}
219
220Value acc::castToI64(Location loc, Value value, OpBuilder &builder) {
221 Type i64Ty = builder.getI64Type();
222 if (value.getType() == i64Ty)
223 return value;
224 if (isa<IndexType>(Val: value.getType()))
225 return arith::IndexCastOp::create(builder, location: loc, out: i64Ty, in: value);
226 unsigned bitwidth = value.getType().getIntOrFloatBitWidth();
227 if (bitwidth > 64)
228 return arith::TruncIOp::create(builder, location: loc, out: i64Ty, in: value);
229 return arith::ExtSIOp::create(builder, location: loc, out: i64Ty, in: value);
230}
231
232Value acc::constantI64(Location loc, int64_t value, OpBuilder &builder) {
233 return LLVM::ConstantOp::create(builder, location: loc, type: builder.getI64Type(), value);
234}
235
236Value acc::remapOrSelf(Value value, ConversionPatternRewriter &rewriter) {
237 if (!value)
238 return {};
239 if (Value converted = rewriter.getRemappedValue(key: value))
240 return converted;
241 return value;
242}
243
244Value acc::getAsyncQueue(Location loc, Value asyncOperand, bool asyncOnly,
245 OpBuilder &builder,
246 const ACCRuntimeCallConfig &config) {
247 Type i64Ty = builder.getI64Type();
248 if (asyncOnly)
249 return LLVM::ConstantOp::create(builder, location: loc, type: i64Ty,
250 value: config.getAsyncNoValueRuntimeValue());
251 if (asyncOperand)
252 return castToI64(loc, value: asyncOperand, builder);
253 return LLVM::ConstantOp::create(builder, location: loc, type: i64Ty,
254 value: config.getAsyncSyncRuntimeValue());
255}
256
257LogicalResult acc::emitWaitCall(Location loc, ValueRange waitOperands,
258 Value asyncQueue, OpBuilder &builder,
259 Region &globalSymbolRegion,
260 SymbolTable &symbolTable,
261 const ACCRuntimeCallConfig &config,
262 Value deviceNum) {
263 Type i32Ty = builder.getI32Type();
264 Type i64Ty = builder.getI64Type();
265 Type ptrTy = LLVM::LLVMPointerType::get(context: builder.getContext());
266
267 SmallVector<Value> queues;
268 queues.reserve(N: waitOperands.size());
269 for (Value waitOperand : waitOperands)
270 queues.push_back(Elt: castToI64(loc, value: waitOperand, builder));
271
272 unsigned size = queues.size();
273 Value waitNum = LLVM::ConstantOp::create(builder, location: loc, type: i32Ty, value: size);
274 Value waitList;
275 if (size == 0) {
276 waitList = LLVM::ZeroOp::create(builder, location: loc, res: ptrTy);
277 } else {
278 waitList = LLVM::AllocaOp::create(builder, location: loc, resultType: ptrTy, elementType: i64Ty, arraySize: waitNum);
279 for (auto [index, queue] : llvm::enumerate(First&: queues)) {
280 Value idx = LLVM::ConstantOp::create(builder, location: loc, type: i32Ty,
281 value: static_cast<int64_t>(index));
282 Value elementPtr = LLVM::GEPOp::create(builder, location: loc, resultType: ptrTy, elementType: i64Ty,
283 basePtr: waitList, indices: ArrayRef<Value>{idx});
284 LLVM::StoreOp::create(builder, location: loc, value: queue, addr: elementPtr);
285 }
286 }
287
288 StringRef functionName = getParentFunctionName(values: waitOperands);
289 if (functionName.empty())
290 if (Block *block = builder.getInsertionBlock())
291 functionName = getParentFunctionName(op: block->getParentOp());
292 Value ident = createIdent(loc, functionName, builder, globalSymbolRegion,
293 config, symbolTable: &symbolTable);
294 Value flags = LLVM::ConstantOp::create(builder, location: loc, type: i64Ty, value: 0);
295 Value deviceType = LLVM::ConstantOp::create(
296 builder, location: loc, type: i64Ty, value: config.getDeviceTypeRuntimeValue(type: DeviceType::None));
297 if (!deviceNum)
298 deviceNum = LLVM::ConstantOp::create(builder, location: loc, type: i32Ty, value: 0);
299
300 return createRuntimeCall(
301 loc, builder, globalSymbolRegion, symbolTable,
302 fn: RuntimeFunction::ACCRTL_tgt_acc_wait, config,
303 arguments: {ident, flags, deviceType, deviceNum, waitNum, waitList, asyncQueue});
304}
305
306FailureOr<Value> acc::emitGetDevicePtrCall(Operation *clauseOp, Value hostPtr,
307 bool ifPresent, OpBuilder &builder,
308 Region &globalSymbolRegion,
309 SymbolTable &symbolTable,
310 const ACCRuntimeCallConfig &config) {
311 Location loc = clauseOp->getLoc();
312 Value ident = createIdent(loc, functionName: getParentFunctionName(op: clauseOp), builder,
313 globalSymbolRegion, config, symbolTable: &symbolTable);
314 Value flags = LLVM::ConstantOp::create(
315 builder, location: loc, type: builder.getI64Type(),
316 value: config.getMapFlagsRuntimeValue(flags: ifPresent ? MapFlags::if_present
317 : MapFlags::none));
318 // Both slots hold the address of the object, a clause naming a section of
319 // one included: the entry point looks up the mapping that address belongs
320 // to rather than being told which one to ask about.
321 FailureOr<LLVM::CallOp> call = createRuntimeCall(
322 loc, builder, globalSymbolRegion, symbolTable,
323 fn: RuntimeFunction::ACCRTL_tgt_acc_get_deviceptr, config,
324 arguments: {ident, /*basePtr=*/hostPtr, flags, /*hostPtr=*/hostPtr});
325 if (failed(Result: call))
326 return failure();
327 return call->getResult();
328}
329
330LogicalResult acc::emitGuardedByIfCond(Location loc, Value ifCond,
331 RewriterBase &rewriter,
332 function_ref<LogicalResult()> emitFn) {
333 if (!ifCond)
334 return emitFn();
335
336 Block *parentBlock = rewriter.getInsertionBlock();
337 Block *continueBlock =
338 rewriter.splitBlock(block: parentBlock, before: rewriter.getInsertionPoint());
339 Block *thenBlock = rewriter.createBlock(
340 parent: parentBlock->getParent(), insertPt: std::next(x: Region::iterator(parentBlock)));
341
342 rewriter.setInsertionPointToEnd(parentBlock);
343 LLVM::CondBrOp::create(builder&: rewriter, location: loc, condition: ifCond, trueDest: thenBlock, trueOperands: ValueRange{},
344 falseDest: continueBlock, falseOperands: ValueRange{});
345
346 rewriter.setInsertionPointToStart(thenBlock);
347 LogicalResult result = emitFn();
348 rewriter.setInsertionPointToEnd(thenBlock);
349 LLVM::BrOp::create(builder&: rewriter, location: loc, operands: ValueRange{}, dest: continueBlock);
350 rewriter.setInsertionPointToStart(continueBlock);
351 return result;
352}
353
354FailureOr<Value>
355acc::emitValueSelectedByIfCond(Location loc, Value ifCond,
356 RewriterBase &rewriter,
357 function_ref<FailureOr<Value>()> thenFn,
358 function_ref<FailureOr<Value>()> elseFn) {
359 if (!ifCond)
360 return thenFn();
361
362 Block *parentBlock = rewriter.getInsertionBlock();
363 Block *continueBlock =
364 rewriter.splitBlock(block: parentBlock, before: rewriter.getInsertionPoint());
365 Block *thenBlock = rewriter.createBlock(
366 parent: parentBlock->getParent(), insertPt: std::next(x: Region::iterator(parentBlock)));
367 Block *elseBlock = rewriter.createBlock(
368 parent: parentBlock->getParent(), insertPt: std::next(x: Region::iterator(thenBlock)));
369
370 rewriter.setInsertionPointToEnd(parentBlock);
371 LLVM::CondBrOp::create(builder&: rewriter, location: loc, condition: ifCond, trueDest: thenBlock, trueOperands: ValueRange{},
372 falseDest: elseBlock, falseOperands: ValueRange{});
373
374 rewriter.setInsertionPointToStart(thenBlock);
375 FailureOr<Value> thenValue = thenFn();
376 if (failed(Result: thenValue))
377 return failure();
378 rewriter.setInsertionPointToEnd(thenBlock);
379 LLVM::BrOp::create(builder&: rewriter, location: loc, operands: ValueRange{*thenValue}, dest: continueBlock);
380
381 rewriter.setInsertionPointToStart(elseBlock);
382 FailureOr<Value> elseValue = elseFn();
383 if (failed(Result: elseValue))
384 return failure();
385 rewriter.setInsertionPointToEnd(elseBlock);
386 LLVM::BrOp::create(builder&: rewriter, location: loc, operands: ValueRange{*elseValue}, dest: continueBlock);
387
388 assert(thenValue->getType() == elseValue->getType() &&
389 "both paths of an if clause have to produce the same type");
390 BlockArgument selected =
391 continueBlock->addArgument(type: thenValue->getType(), loc);
392 rewriter.setInsertionPointToStart(continueBlock);
393 return Value(selected);
394}
395
396void acc::spliceConstructRegion(Operation *op, Region &region,
397 RewriterBase &rewriter) {
398 Block *prev = op->getBlock();
399 Block *succ = rewriter.splitBlock(block: prev, before: Block::iterator(op));
400 rewriter.setInsertionPointToEnd(prev);
401 LLVM::BrOp::create(builder&: rewriter, location: op->getLoc(), dest: &region.getBlocks().front());
402 for (Block &block : region.getBlocks()) {
403 // A region that ends without a terminator, as an `acc.kernel_environment`
404 // body does, continues after the construct from the end of its block.
405 if (block.empty() || !block.back().hasTrait<OpTrait::IsTerminator>()) {
406 rewriter.setInsertionPointToEnd(&block);
407 LLVM::BrOp::create(builder&: rewriter, location: op->getLoc(), dest: succ);
408 continue;
409 }
410 Operation *terminator = block.getTerminator();
411 if (isa<acc::TerminatorOp>(Val: terminator)) {
412 rewriter.setInsertionPoint(terminator);
413 LLVM::BrOp::create(builder&: rewriter, location: op->getLoc(), dest: succ);
414 rewriter.eraseOp(op: terminator);
415 }
416 }
417 rewriter.inlineRegionBefore(region, before: succ);
418}
419