1//===- LoweringPrepare.cpp - pareparation work for LLVM lowering ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "PassDetail.h"
10#include "TargetLowering/LowerModule.h"
11#include "mlir/IR/Attributes.h"
12#include "mlir/IR/BuiltinAttributeInterfaces.h"
13#include "mlir/IR/IRMapping.h"
14#include "mlir/IR/Location.h"
15#include "mlir/IR/Value.h"
16#include "clang/Basic/Cuda.h"
17#include "clang/Basic/SourceManager.h"
18#include "clang/Basic/Specifiers.h"
19#include "clang/Basic/TargetCXXABI.h"
20#include "clang/Basic/TargetInfo.h"
21#include "clang/CIR/Dialect/Builder/CIRBaseBuilder.h"
22#include "clang/CIR/Dialect/IR/CIRAttrs.h"
23#include "clang/CIR/Dialect/IR/CIRDataLayout.h"
24#include "clang/CIR/Dialect/IR/CIRDialect.h"
25#include "clang/CIR/Dialect/IR/CIROpsEnums.h"
26#include "clang/CIR/Dialect/IR/CIRTypes.h"
27#include "clang/CIR/Dialect/Passes.h"
28#include "clang/CIR/Dialect/Transforms/CIRTransformUtils.h"
29#include "clang/CIR/Interfaces/ASTAttrInterfaces.h"
30#include "clang/CIR/MissingFeatures.h"
31#include "llvm/ADT/StringRef.h"
32#include "llvm/ADT/TypeSwitch.h"
33#include "llvm/IR/Instructions.h"
34#include "llvm/Support/ErrorHandling.h"
35#include "llvm/Support/Path.h"
36#include "llvm/Support/VersionTuple.h"
37
38#include <map>
39#include <memory>
40#include <optional>
41
42using namespace mlir;
43using namespace cir;
44
45namespace mlir {
46#define GEN_PASS_DEF_LOWERINGPREPARE
47#include "clang/CIR/Dialect/Passes.h.inc"
48} // namespace mlir
49
50static SmallString<128> getTransformedFileName(mlir::ModuleOp mlirModule) {
51 SmallString<128> fileName;
52
53 if (mlirModule.getSymName())
54 fileName = llvm::sys::path::filename(path: mlirModule.getSymName()->str());
55
56 if (fileName.empty())
57 fileName = "<null>";
58
59 for (size_t i = 0; i < fileName.size(); ++i) {
60 // Replace everything that's not [a-zA-Z0-9._] with a _. This set happens
61 // to be the set of C preprocessing numbers.
62 if (!clang::isPreprocessingNumberBody(c: fileName[i]))
63 fileName[i] = '_';
64 }
65
66 return fileName;
67}
68
69namespace {
70struct LoweringPreparePass
71 : public impl::LoweringPrepareBase<LoweringPreparePass> {
72 LoweringPreparePass() = default;
73
74 // `mlir::SymbolTableCollection` is move-only (it owns lazily-created
75 // `unique_ptr<SymbolTable>` entries), which makes the implicit copy
76 // constructor ill-formed. MLIR's `clonePass()` requires copy
77 // construction, so define one explicitly. Per-run state members
78 // (dynamic initializers, guard maps, symbol-table cache, etc.) all
79 // start fresh in the cloned pass, which matches MLIR convention for
80 // pass clones and is more correct than the previous default-generated
81 // behavior that silently copied them.
82 LoweringPreparePass(const LoweringPreparePass &other)
83 : impl::LoweringPrepareBase<LoweringPreparePass>(other) {}
84
85 void runOnOperation() override;
86
87 void runOnOp(mlir::Operation *op);
88 void lowerCastOp(cir::CastOp op);
89 void lowerComplexConjOp(cir::ComplexConjOp op);
90 void lowerComplexDivOp(cir::ComplexDivOp op);
91 void lowerComplexMulOp(cir::ComplexMulOp op);
92 void lowerComplexFDivOp(cir::ComplexFDivOp op);
93 void lowerComplexFMulOp(cir::ComplexFMulOp op);
94 void lowerGetGlobalOp(cir::GetGlobalOp op);
95 void lowerGlobalOp(cir::GlobalOp op);
96 void lowerThreeWayCmpOp(cir::CmpThreeWayOp op);
97 void lowerArrayDtor(cir::ArrayDtor op);
98 void lowerArrayCtor(cir::ArrayCtor op);
99 void lowerTrivialCopyCall(cir::CallOp op);
100 void lowerStoreOfConstAggregate(cir::StoreOp op);
101 void lowerLocalInitOp(cir::LocalInitOp op);
102 void lowerRegisterExitDtorOp(cir::RegisterExitDtorOp op);
103 void lowerStdOp(cir::StdOpInterface op);
104
105 /// Return the FuncOp called by `callOp`. Uses the cached `symbolTables`
106 /// member to avoid the O(M) module-wide scan that the static
107 /// `mlir::SymbolTable::lookupNearestSymbolFrom` would do per call.
108 cir::FuncOp getCalledFunction(cir::CallOp callOp);
109
110 /// Return a private constant cir::GlobalOp with the given type and initial
111 /// value, suitable for backing a memcpy-initialized local aggregate.
112 ///
113 /// If a global with `baseName` (or one of its `.<n>` versioned siblings)
114 /// already has a matching type and initial value, that global is reused.
115 /// Otherwise a new global is created with the next available `.<n>` suffix
116 /// (matching CIRGenBuilder::createVersionedGlobal and OGCG behavior).
117 cir::GlobalOp
118 getOrCreateConstAggregateGlobal(CIRBaseBuilderTy &builder, mlir::Location loc,
119 llvm::StringRef baseName, mlir::Type ty,
120 mlir::TypedAttr constant, uint64_t alignment);
121
122 /// Build the function that initializes the specified global
123 cir::FuncOp buildCXXGlobalVarDeclInitFunc(cir::GlobalOp op);
124
125 /// When looking at the 'global' op, create the wrapper function.
126 void defineGlobalThreadLocalWrapper(cir::GlobalOp op, cir::FuncOp initAlias,
127 bool isVarDefinition);
128 /// Create an initialization alias for a thread-local variable.
129 cir::FuncOp defineGlobalThreadLocalInitAlias(cir::GlobalOp op,
130 cir::FuncOp aliasee);
131 /// Get the declaration for the 'wrapper' function for a global-TLS variable.
132 cir::FuncOp getOrCreateThreadLocalWrapper(CIRBaseBuilderTy &builder,
133 cir::GlobalOp op);
134 // Function that generates the get-global and 'if' condition on a guard
135 // variable for an init function that must run once: the global TLS init
136 // function, or the initializer of a C++20 named-module interface unit. This
137 // inserts an 'if' with the store at the beginning of the 'then' region, so
138 // inserts into the body should happen after that.
139 cir::IfOp buildInitGuardCheck(CIRBaseBuilderTy &builder, mlir::Location loc,
140 cir::GlobalOp guard, bool threadLocal);
141 /// Handle the dtor region by registering destructor with __cxa_atexit
142 cir::FuncOp getOrCreateDtorFunc(CIRBaseBuilderTy &builder, cir::GlobalOp op,
143 mlir::Region &dtorRegion,
144 cir::CallOp &dtorCall);
145 /// Create a `void __cxx_global_array_dtor[.N](void *)` helper after `op`,
146 /// with an empty entry block.
147 cir::FuncOp buildGlobalArrayDtorFunc(CIRBaseBuilderTy &builder,
148 mlir::Operation *op);
149
150 /// Build a function named `fnName` with the given linkage that calls each
151 /// of `initializers` in order, then returns, and register it in
152 /// `globalCtorList` under `priority`. With a `guard` byte, the calls run
153 /// only when it is still zero and set it first. Shared by the default
154 /// `_GLOBAL__sub_I_*` initializer function, the C++20 named-module
155 /// initializer, and the per-priority `_GLOBAL__I_<priority>` initializer
156 /// functions.
157 cir::FuncOp
158 buildGlobalInitCallerFunc(llvm::StringRef fnName,
159 cir::GlobalLinkageKind linkage,
160 llvm::ArrayRef<cir::FuncOp> initializers,
161 uint32_t priority, cir::GlobalOp guard = {});
162
163 /// Build a module init function that calls all the dynamic initializers.
164 void buildCXXGlobalInitFunc();
165 /// Build one `_GLOBAL__I_<priority>` function per distinct priority found
166 /// in `prioritizedDynamicInitializers`, in ascending priority order, and
167 /// register each with `globalCtorList`. The first of them, the highest
168 /// priority, calls `importedInits` before its own initializers and takes
169 /// them out of the vector, as classic CodeGen does.
170 void buildCXXGlobalPriorityInitFuncs(
171 llvm::SmallVectorImpl<cir::FuncOp> &importedInits);
172 // Build an init function for all of the ordered global thread local storage
173 // variables.
174 void buildCXXGlobalTlsFunc();
175
176 /// Materialize global ctor/dtor list
177 void buildGlobalCtorDtorList();
178
179 /// Init and cleanup functions pass \p hasDefaultFuncAttrs true.
180 cir::FuncOp buildRuntimeFunction(
181 mlir::OpBuilder &builder, llvm::StringRef name, mlir::Location loc,
182 cir::FuncType type,
183 cir::GlobalLinkageKind linkage = cir::GlobalLinkageKind::ExternalLinkage,
184 bool hasDefaultFuncAttrs = false);
185
186 cir::GlobalOp getOrCreateRuntimeVariable(
187 mlir::OpBuilder &builder, llvm::StringRef name, mlir::Location loc,
188 mlir::Type type,
189 cir::GlobalLinkageKind linkage = cir::GlobalLinkageKind::ExternalLinkage,
190 cir::VisibilityKind visibility = cir::VisibilityKind::Default);
191
192 /// ------------
193 /// CUDA registration related
194 /// ------------
195
196 llvm::StringMap<FuncOp> cudaKernelMap;
197 llvm::SmallVector<std::pair<cir::GlobalOp, cir::CUDAVarRegistrationInfoAttr>>
198 cudaDeviceVars;
199
200 /// Build the CUDA module constructor that registers the fat binary
201 /// with the CUDA runtime.
202 void buildCUDAModuleCtor();
203 std::optional<FuncOp> buildCUDAModuleDtor();
204 std::optional<FuncOp> buildHIPModuleDtor();
205 std::optional<FuncOp> buildCUDARegisterGlobals();
206 void buildCUDARegisterVars(cir::CIRBaseBuilderTy &builder,
207 FuncOp regGlobalFunc);
208 void buildCUDARegisterGlobalFunctions(cir::CIRBaseBuilderTy &builder,
209 FuncOp regGlobalFunc);
210
211 /// Handle static local variable initialization with guard variables.
212 void handleStaticLocal(cir::GlobalOp globalOp, cir::LocalInitOp localInitOp);
213
214 /// Handle the guard-protected init for static local/weak/linkonce-odr
215 /// variables.
216 void emitGuardedInit(CIRBaseBuilderTy &builder, cir::GlobalOp globalOp,
217 mlir::Region &ctorRegion, mlir::Region &dtorRegion);
218
219 /// Get or create __cxa_guard_acquire function.
220 cir::FuncOp getGuardAcquireFn(cir::PointerType guardPtrTy);
221
222 /// Get or create __cxa_guard_release function.
223 cir::FuncOp getGuardReleaseFn(cir::PointerType guardPtrTy);
224
225 /// Get or create __cxa_guard_abort function.
226 cir::FuncOp getGuardAbortFn(cir::PointerType guardPtrTy);
227
228 /// Get or create the __init_tls function.
229 cir::FuncOp getTlsInitFn();
230
231 // Create the __tls_guard variable.
232 cir::GlobalOp createGlobalThreadLocalGuard(CIRBaseBuilderTy &builder,
233 mlir::Location loc);
234
235 /// Create a guard global variable for a static local.
236 cir::GlobalOp createGuardGlobalOp(CIRBaseBuilderTy &builder,
237 mlir::Location loc, llvm::StringRef name,
238 cir::IntType guardTy,
239 cir::GlobalLinkageKind linkage);
240
241 /// Append 'global' to the module's llvm.used list to make sure it doesn't get
242 /// garbage-collected by the linker.
243 void addUsedGlobal(cir::GlobalOp global);
244
245 /// Get the guard variable for a static local declaration.
246 cir::GlobalOp getStaticLocalDeclGuardAddress(llvm::StringRef globalSymName) {
247 auto it = staticLocalDeclGuardMap.find(Key: globalSymName);
248 if (it != staticLocalDeclGuardMap.end())
249 return it->second;
250 return nullptr;
251 }
252
253 /// Set the guard variable for a static local declaration.
254 void setStaticLocalDeclGuardAddress(llvm::StringRef globalSymName,
255 cir::GlobalOp guard) {
256 staticLocalDeclGuardMap[globalSymName] = guard;
257 }
258
259 /// Get or create the guard variable for a static local declaration.
260 cir::GlobalOp getOrCreateStaticLocalDeclGuardAddress(
261 CIRBaseBuilderTy &builder, cir::GlobalOp globalOp, StringRef guardName,
262 bool isLocalVarDecl, bool useInt8GuardVariable) {
263
264 cir::CIRDataLayout dataLayout(mlirModule);
265 cir::IntType guardTy;
266 clang::CharUnits guardAlignment;
267 // Guard variables are 64 bits in the generic ABI and size width on ARM
268 // (i.e. 32-bit on AArch32, 64-bit on AArch64).
269 if (useInt8GuardVariable) {
270 guardTy = cir::IntType::get(context: &getContext(), width: 8, /*isSigned=*/is_signed: true);
271 guardAlignment = clang::CharUnits::One();
272 } else if (useARMGuardVarABI()) {
273 // Guard variables are size width on ARM (32-bit AArch32, 64-bit AArch64).
274 const unsigned sizeTypeSize =
275 getTargetInfo().getTypeWidth(T: getTargetInfo().getSignedSizeType());
276 guardTy =
277 cir::IntType::get(context: &getContext(), width: sizeTypeSize, /*isSigned=*/is_signed: true);
278 guardAlignment =
279 clang::CharUnits::fromQuantity(Quantity: dataLayout.getABITypeAlign(ty: guardTy));
280 } else {
281 guardTy = cir::IntType::get(context: &getContext(), width: 64, /*isSigned=*/is_signed: true);
282 guardAlignment =
283 clang::CharUnits::fromQuantity(Quantity: dataLayout.getABITypeAlign(ty: guardTy));
284 }
285 assert(guardTy && guardAlignment.getQuantity() != 0);
286
287 llvm::StringRef globalSymName = globalOp.getSymName();
288 cir::GlobalOp guard = getStaticLocalDeclGuardAddress(globalSymName);
289 if (!guard) {
290 // Create the guard variable with a zero-initializer.
291 guard = createGuardGlobalOp(builder, loc: globalOp->getLoc(), name: guardName,
292 guardTy, linkage: globalOp.getLinkage());
293 guard.setInitialValueAttr(cir::IntAttr::get(type: guardTy, value: 0));
294 guard.setDSOLocal(globalOp.getDsoLocal());
295 guard.setAlignment(guardAlignment.getAsAlign().value());
296 guard.setTlsModel(globalOp.getTlsModel());
297 guard.setGlobalVisibility(globalOp.getGlobalVisibility());
298
299 // The ABI says: "It is suggested that it be emitted in the same COMDAT
300 // group as the associated data object." In practice, this doesn't work
301 // for non-ELF and non-Wasm object formats, so only do it for ELF and
302 // Wasm.
303 std::optional<llvm::StringRef> comdat = globalOp.getComdat();
304 const llvm::Triple &triple = getTargetInfo().getTriple();
305 if (!isLocalVarDecl && comdat.has_value() &&
306 (triple.isOSBinFormatELF() || triple.isOSBinFormatWasm())) {
307 guard.setComdat(comdat->empty() ? globalOp.getSymName() : *comdat);
308 } else if (triple.supportsCOMDAT() && guard.isWeakForLinker()) {
309 // A weak guard that isn't joining the variable's COMDAT gets its own,
310 // even if the variable has none (such as a plain weak variable).
311 guard.setSelfComdat();
312 }
313
314 setStaticLocalDeclGuardAddress(globalSymName, guard);
315 }
316 return guard;
317 }
318
319 /// Target/ABI facts sourced from the module's own attributes.
320 std::unique_ptr<cir::LowerModule> lowerModule;
321
322 const clang::TargetInfo &getTargetInfo() const {
323 assert(lowerModule && "LoweringPrepare requires a module with a triple");
324 return lowerModule->getTarget();
325 }
326
327 const clang::LangOptions &getLangOpts() const {
328 assert(lowerModule && "LoweringPrepare requires a module with LangOptions");
329 return lowerModule->getLangOpts();
330 }
331
332 /// Platform SDK version recorded on the module by CIRGen. An absent attribute
333 /// yields an empty VersionTuple, which is what a compilation without
334 /// -target-sdk-version behaves like: no version-gated feature is enabled.
335 /// Returns std::nullopt if the attribute is present but unparseable; an error
336 /// has been emitted and the pass marked as failed, so the caller must bail
337 /// out.
338 std::optional<llvm::VersionTuple> getSDKVersion() {
339 auto sdkVersionAttr = mlirModule->getAttrOfType<mlir::StringAttr>(
340 name: CIRDialect::getSDKVersionAttrName());
341 if (!sdkVersionAttr)
342 return llvm::VersionTuple();
343
344 llvm::VersionTuple sdkVersion;
345 if (sdkVersion.tryParse(string: sdkVersionAttr.getValue())) {
346 mlirModule->emitError(message: "cannot parse platform SDK version from ")
347 << CIRDialect::getSDKVersionAttrName() << " = '"
348 << sdkVersionAttr.getValue() << "'";
349 signalPassFailure();
350 return std::nullopt;
351 }
352 return sdkVersion;
353 }
354
355 /// Appends the CUID hash recorded on the module by CIRGen, if any, to
356 /// \p name.
357 std::string addCUIDSuffix(llvm::StringRef name) {
358 std::string result = name.str();
359 if (auto cuidHash = mlirModule->getAttrOfType<mlir::StringAttr>(
360 name: CIRDialect::getCUIDHashAttrName()))
361 result += ("_" + cuidHash.getValue()).str();
362 return result;
363 }
364
365 /// Tracks current module.
366 mlir::ModuleOp mlirModule;
367
368 /// Cached symbol tables used to avoid repeated O(M) module-wide scans
369 /// during per-call/per-global symbol lookups. Lazily populated on first
370 /// use. Pass methods access this directly rather than threading it
371 /// through helper signatures (see PR feedback on #195919).
372 ///
373 /// Invariant: every site that mutates the module's symbol table either
374 /// (a) keeps `symbolTables` in sync via
375 /// `symbolTables.getSymbolTable(mlirModule).insert(...)` (as
376 /// `getOrCreateConstAggregateGlobal` does), or (b) creates a symbol
377 /// that is never resolved through the cache later. Today
378 /// `buildRuntimeFunction` and `getOrCreateRuntimeVariable` fall in the
379 /// (b) bucket: their callers either use a separate map
380 /// (`cudaKernelMap`, `staticLocalDeclGuardMap`, `dynamicInitializers`)
381 /// or the static `mlir::SymbolTable::lookupNearestSymbolFrom`, never
382 /// the cached path. If a future change adds a cached lookup of a
383 /// freshly created symbol, the corresponding create site MUST move
384 /// to bucket (a) (insert into the cache or call
385 /// `invalidateSymbolTable`).
386 mlir::SymbolTableCollection symbolTables;
387
388 /// Tracks existing dynamic initializers.
389 llvm::StringMap<uint32_t> dynamicInitializerNames;
390 llvm::SmallVector<cir::FuncOp> dynamicInitializers;
391 /// Dynamic initializers with an explicit `init_priority` attribute,
392 /// grouped by priority (keys kept in ascending order). Each group is
393 /// emitted into its own `_GLOBAL__I_<priority>` function instead of being
394 /// folded into the single default-priority `_GLOBAL__sub_I_*` function.
395 std::map<unsigned, llvm::SmallVector<cir::FuncOp, 4>>
396 prioritizedDynamicInitializers;
397 llvm::SmallVector<cir::FuncOp> globalThreadLocalInitializers;
398 llvm::StringMap<cir::FuncOp> threadLocalWrappers;
399 llvm::StringMap<cir::FuncOp> threadLocalInitAliases;
400
401 /// Tracks guard variables for static locals (keyed by global symbol name).
402 llvm::StringMap<cir::GlobalOp> staticLocalDeclGuardMap;
403
404 llvm::StringMap<llvm::SmallVector<cir::GlobalOp, 1>> constAggregateGlobals;
405
406 /// List of ctors and their priorities to be called before main()
407 llvm::SmallVector<cir::GlobalCtorDtorEntry, 4> globalCtorList;
408 /// List of dtors and their priorities to be called when unloading module.
409 llvm::SmallVector<cir::GlobalCtorDtorEntry, 4> globalDtorList;
410
411 /// Returns true if the target uses ARM-style guard variables for static
412 /// local initialization (32-bit guard, check bit 0 only).
413 bool useARMGuardVarABI() const {
414 switch (lowerModule->getCXXABIKind()) {
415 case clang::TargetCXXABI::GenericARM:
416 case clang::TargetCXXABI::iOS:
417 case clang::TargetCXXABI::WatchOS:
418 case clang::TargetCXXABI::GenericAArch64:
419 case clang::TargetCXXABI::WebAssembly:
420 return true;
421 default:
422 return false;
423 }
424 }
425
426 /// Register `dtorFunc` to be called with `arg` when the program exits, or
427 /// when the current thread exits if `tls` is set.
428 cir::CallOp emitAtExitCall(CIRBaseBuilderTy &builder, mlir::Location loc,
429 cir::FuncOp dtorFunc, mlir::Value arg, bool tls) {
430 cir::GlobalOp handle;
431 cir::FuncOp fnAtExit;
432 cir::PointerType voidPtrTy = builder.getVoidPtrTy();
433 cir::PointerType voidFnPtrTy = builder.getVoidFnPtrTy(argTypes: {voidPtrTy});
434 {
435 mlir::OpBuilder::InsertionGuard guard(builder);
436 // Create a variable that binds the atexit to this shared object.
437 builder.setInsertionPointToStart(&mlirModule.getBodyRegion().front());
438 handle = getOrCreateRuntimeVariable(
439 builder, name: "__dso_handle", loc, type: builder.getUIntNTy(n: 8),
440 linkage: cir::GlobalLinkageKind::ExternalLinkage, visibility: cir::VisibilityKind::Hidden);
441
442 // Create a runtime helper function:
443 // extern "C" int __cxa_atexit(void (*f)(void *), void *p, void *d);
444 cir::PointerType handlePtrTy = builder.getPointerTo(ty: handle.getSymType());
445 IntType intTy = builder.getSIntNTy(n: 32);
446 auto fnAtExitType =
447 cir::FuncType::get(inputs: {voidFnPtrTy, voidPtrTy, handlePtrTy}, returnType: intTy);
448
449 llvm::StringLiteral nameAtExit = "__cxa_atexit";
450 if (tls)
451 nameAtExit = getTargetInfo().getTriple().isOSDarwin()
452 ? llvm::StringLiteral("_tlv_atexit")
453 : llvm::StringLiteral("__cxa_thread_atexit");
454
455 fnAtExit = buildRuntimeFunction(builder, name: nameAtExit, loc, type: fnAtExitType);
456 }
457
458 mlir::Value args[3];
459 auto dtorPtrTy = cir::PointerType::get(pointee: dtorFunc.getFunctionType());
460 args[0] = cir::GetGlobalOp::create(builder, location: loc, addr: dtorPtrTy,
461 name: dtorFunc.getSymName());
462 args[0] = cir::CastOp::create(builder, location: loc, result: voidFnPtrTy,
463 kind: cir::CastKind::bitcast, src: args[0]);
464 args[1] = cir::CastOp::create(builder, location: loc, result: voidPtrTy,
465 kind: cir::CastKind::bitcast, src: arg);
466 args[2] = cir::GetGlobalOp::create(
467 builder, location: loc, addr: builder.getPointerTo(ty: handle.getSymType()),
468 name: handle.getSymName());
469 // TODO(cir): set the runtime calling convention on the __cxa_atexit call.
470 assert(!cir::MissingFeatures::opFuncCallingConv());
471 return builder.createCallOp(loc, callee: fnAtExit, operands: args);
472 }
473
474 void emitGlobalGuardedDtorRegion(CIRBaseBuilderTy &builder,
475 cir::GlobalOp global,
476 mlir::Region &dtorRegion, bool tls,
477 mlir::Block &entryBB) {
478 // If this is a simple call to a destructor, get the called function.
479 // Otherwise, create a helper function for the entire dtor region,
480 // replacing the current dtor region body with a call to the helper
481 // function.
482 cir::CallOp dtorCall;
483 cir::FuncOp dtorFunc =
484 getOrCreateDtorFunc(builder, op: global, dtorRegion, dtorCall);
485
486 // Replace the dtor (or helper) call with a call to
487 // __cxa_atexit(&dtor, &var, &__dso_handle)
488 builder.setInsertionPointAfter(dtorCall);
489 emitAtExitCall(builder, loc: dtorCall.getLoc(), dtorFunc,
490 arg: dtorCall.getArgOperand(i: 0), tls);
491 dtorCall->erase();
492 mlir::Block &dtorBlock = dtorRegion.front();
493 entryBB.getOperations().splice(where: entryBB.end(), L2&: dtorBlock.getOperations(),
494 first: dtorBlock.begin(),
495 last: std::prev(x: dtorBlock.end()));
496 // make sure we leave the insert location after the operations we just
497 // inserted.
498 builder.setInsertionPointToEnd(&entryBB);
499 }
500
501 /// Emit the guarded initialization for a static local variable.
502 /// This handles the if/else structure after the guard byte check,
503 /// following OG's ItaniumCXXABI::EmitGuardedInit skeleton.
504 void emitCXXGuardedInitIf(CIRBaseBuilderTy &builder, cir::GlobalOp globalOp,
505 mlir::Region &ctorRegion, mlir::Region &dtorRegion,
506 bool isLocalVarDecl, mlir::Value guardPtr,
507 cir::PointerType guardPtrTy, bool threadsafe) {
508 auto loc = globalOp->getLoc();
509
510 // TODO(cir): set the runtime calling convention on the __cxa_guard_* calls.
511 assert(!cir::MissingFeatures::opFuncCallingConv());
512
513 // The semantics of dynamic initialization of variables with static or
514 // thread storage duration depends on whether they are declared at
515 // block-scope. The initialization of such variables at block-scope can be
516 // aborted with an exception and later retried (per C++20 [stmt.dcl]p4),
517 // and recursive entry to their initialization has undefined behavior (also
518 // per C++20 [stmt.dcl]p4). For such variables declared at non-block scope,
519 // exceptions lead to termination (per C++20 [except.terminate]p1), and
520 // recursive references to the variables are governed only by the lifetime
521 // rules (per C++20 [class.cdtor]p2), which means such references are
522 // perfectly fine as long as they avoid touching memory. As a result,
523 // block-scope variables must not be marked as initialized until after
524 // initialization completes (unless the mark is reverted following an
525 // exception), but non-block-scope variables must be marked prior to
526 // initialization so that recursive accesses during initialization do not
527 // restart initialization.
528
529 auto emitBody = [&]() {
530 // Emit the initializer and add a global destructor if appropriate.
531 mlir::Block *insertBlock = builder.getInsertionBlock();
532 if (!ctorRegion.empty()) {
533 assert(ctorRegion.hasOneBlock() && "Enforced by MaxSizedRegion<1>");
534
535 mlir::Block &block = ctorRegion.front();
536 insertBlock->getOperations().splice(
537 where: insertBlock->end(), L2&: block.getOperations(), first: block.begin(),
538 last: std::prev(x: block.end()));
539 }
540
541 if (!dtorRegion.empty()) {
542 assert(dtorRegion.hasOneBlock() && "Enforced by MaxSizedRegion<1>");
543
544 // Whether the destructor is registered with __cxa_thread_atexit depends
545 // on the variable being thread_local, not on the guard being
546 // thread-safe, so make sure we use getTlsModel here instead of
547 // threadsafe.
548 emitGlobalGuardedDtorRegion(builder, global: globalOp, dtorRegion,
549 tls: globalOp.getTlsModel().has_value(),
550 entryBB&: *insertBlock);
551 }
552 builder.setInsertionPointToEnd(insertBlock);
553 ctorRegion.getBlocks().clear();
554 };
555
556 // Variables used when coping with thread-safe statics and exceptions.
557 if (threadsafe) {
558 // Call __cxa_guard_acquire.
559 cir::CallOp acquireCall = builder.createCallOp(
560 loc, callee: getGuardAcquireFn(guardPtrTy), operands: mlir::ValueRange{guardPtr});
561 mlir::Value acquireResult = acquireCall.getResult();
562
563 auto acquireZero = builder.getConstantInt(
564 loc, ty: mlir::cast<cir::IntType>(Val: acquireResult.getType()), value: 0);
565 auto shouldInit = builder.createCompare(loc, kind: cir::CmpOpKind::ne,
566 lhs: acquireResult, rhs: acquireZero);
567
568 // Create the IfOp for the shouldInit check.
569 // Pass an empty callback to avoid auto-creating a yield terminator.
570 auto ifOp =
571 cir::IfOp::create(builder, location: loc, cond: shouldInit, /*withElseRegion=*/false,
572 thenBuilder: [](mlir::OpBuilder &, mlir::Location) {});
573 mlir::OpBuilder::InsertionGuard insertGuard(builder);
574 builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
575
576 // An exception out of the initializer leaves the variable uninitialized
577 // and the next entry has to run the initializer again ([stmt.dcl]p4), so
578 // the guard has to be given back along the exceptional edge. Without
579 // this the guard stays held and that retry terminates instead.
580 //
581 // OG pushes an EH cleanup here and pops it after the initializer, which
582 // costs nothing when there is no unwind path. A cir.cleanup.scope is
583 // structural, so it is only worth building when there can be one.
584 // OG: CGF.EHStack.pushCleanup<CallGuardAbort>(EHCleanup, guard);
585 // ... CGF.PopCleanupBlock();
586 if (getLangOpts().Exceptions) {
587 cir::CleanupScopeOp::create(
588 builder, location: loc, cleanupKind: cir::CleanupKind::EH,
589 bodyBuilder: [&](mlir::OpBuilder &, mlir::Location bodyLoc) {
590 emitBody();
591 builder.createYield(loc: bodyLoc);
592 },
593 cleanupBuilder: [&](mlir::OpBuilder &, mlir::Location cleanupLoc) {
594 cir::CallOp abortCall =
595 builder.createCallOp(loc: cleanupLoc, callee: getGuardAbortFn(guardPtrTy),
596 operands: mlir::ValueRange{guardPtr});
597 // __cxa_guard_abort is declared noexcept, and marking it so keeps
598 // this out of an invoke: a cleanup that can throw while unwinding
599 // needs a terminate edge, which is dead weight here.
600 abortCall.setNothrowAttr(builder.getUnitAttr());
601 builder.createYield(loc: cleanupLoc);
602 });
603 builder.setInsertionPointToEnd(&ifOp.getThenRegion().front());
604 } else {
605 emitBody();
606 }
607
608 // Call __cxa_guard_release. This cannot throw.
609 builder.createCallOp(loc, callee: getGuardReleaseFn(guardPtrTy),
610 operands: mlir::ValueRange{guardPtr});
611
612 builder.createYield(loc);
613 } else if (!isLocalVarDecl) {
614 // For non-local variables, store 1 into the first byte of the guard
615 // variable before the object initialization begins so that references
616 // to the variable during initialization don't restart initialization.
617 mlir::Value byteGuardPtr = builder.createBitcast(
618 src: guardPtr, newTy: cir::PointerType::get(pointee: builder.getSIntNTy(n: 8)));
619 builder.createStore(loc,
620 val: builder.getConstantInt(loc, ty: builder.getSIntNTy(n: 8), value: 1),
621 dst: byteGuardPtr);
622 emitBody();
623 } else {
624 emitBody();
625 // For local variables, store 1 into the first byte of the guard variable
626 // after the object initialization completes so that initialization is
627 // retried if initialization is interrupted by an exception.
628 mlir::Value byteGuardPtr = builder.createBitcast(
629 src: guardPtr, newTy: cir::PointerType::get(pointee: builder.getSIntNTy(n: 8)));
630 builder.createStore(loc,
631 val: builder.getConstantInt(loc, ty: builder.getSIntNTy(n: 8), value: 1),
632 dst: byteGuardPtr);
633 }
634
635 builder.createYield(loc); // Outermost IfOp
636 }
637};
638
639} // namespace
640
641cir::GlobalOp LoweringPreparePass::getOrCreateRuntimeVariable(
642 mlir::OpBuilder &builder, llvm::StringRef name, mlir::Location loc,
643 mlir::Type type, cir::GlobalLinkageKind linkage,
644 cir::VisibilityKind visibility) {
645 cir::GlobalOp g = dyn_cast_or_null<cir::GlobalOp>(
646 Val: mlir::SymbolTable::lookupNearestSymbolFrom(
647 from: mlirModule, symbol: mlir::StringAttr::get(context: mlirModule->getContext(), bytes: name)));
648 if (!g) {
649 g = cir::GlobalOp::create(builder, location: loc, sym_name: name, sym_type: type);
650 g.setLinkageAttr(
651 cir::GlobalLinkageKindAttr::get(context: builder.getContext(), value: linkage));
652 mlir::SymbolTable::setSymbolVisibility(
653 symbol: g, vis: mlir::SymbolTable::Visibility::Private);
654 g.setGlobalVisibility(visibility);
655 }
656 return g;
657}
658
659cir::FuncOp LoweringPreparePass::buildRuntimeFunction(
660 mlir::OpBuilder &builder, llvm::StringRef name, mlir::Location loc,
661 cir::FuncType type, cir::GlobalLinkageKind linkage,
662 bool hasDefaultFuncAttrs) {
663 cir::FuncOp f = dyn_cast_or_null<FuncOp>(Val: SymbolTable::lookupNearestSymbolFrom(
664 from: mlirModule, symbol: StringAttr::get(context: mlirModule->getContext(), bytes: name)));
665 if (!f) {
666 f = cir::FuncOp::create(builder, location: loc, sym_name: name, type);
667 f.setLinkageAttr(
668 cir::GlobalLinkageKindAttr::get(context: builder.getContext(), value: linkage));
669 mlir::SymbolTable::setSymbolVisibility(
670 symbol: f, vis: mlir::SymbolTable::Visibility::Private);
671 if (!hasDefaultFuncAttrs) {
672 f->setAttr(name: cir::CIRDialect::getRuntimeFuncAttrName(),
673 value: builder.getUnitAttr());
674 } else {
675 // Give the function the TU-wide target-cpu/target-features that CIRGen
676 // recorded on the module, as classic codegen does for every function.
677 for (llvm::StringRef name :
678 {cir::CIRDialect::getTargetCPUAttrName(),
679 cir::CIRDialect::getTuneCPUAttrName(),
680 cir::CIRDialect::getTargetFeaturesAttrName()})
681 if (mlir::Attribute attr = mlirModule->getAttr(name))
682 f->setAttr(name, value: attr);
683 }
684
685 assert(!cir::MissingFeatures::opFuncExtraAttrs());
686 }
687 return f;
688}
689
690static mlir::Value lowerScalarToComplexCast(mlir::MLIRContext &ctx,
691 cir::CastOp op) {
692 cir::CIRBaseBuilderTy builder(ctx);
693 builder.setInsertionPoint(op);
694
695 mlir::Value src = op.getSrc();
696 mlir::Value imag = builder.getNullValue(ty: src.getType(), loc: op.getLoc());
697 return builder.createComplexCreate(loc: op.getLoc(), real: src, imag);
698}
699
700static mlir::Value lowerComplexToScalarCast(mlir::MLIRContext &ctx,
701 cir::CastOp op,
702 cir::CastKind elemToBoolKind) {
703 cir::CIRBaseBuilderTy builder(ctx);
704 builder.setInsertionPoint(op);
705
706 mlir::Value src = op.getSrc();
707 if (!mlir::isa<cir::BoolType>(Val: op.getType()))
708 return builder.createComplexReal(loc: op.getLoc(), operand: src);
709
710 // Complex cast to bool: (bool)(a+bi) => (bool)a || (bool)b
711 mlir::Value srcReal = builder.createComplexReal(loc: op.getLoc(), operand: src);
712 mlir::Value srcImag = builder.createComplexImag(loc: op.getLoc(), operand: src);
713
714 cir::BoolType boolTy = builder.getBoolTy();
715 mlir::Value srcRealToBool =
716 builder.createCast(loc: op.getLoc(), kind: elemToBoolKind, src: srcReal, newTy: boolTy);
717 mlir::Value srcImagToBool =
718 builder.createCast(loc: op.getLoc(), kind: elemToBoolKind, src: srcImag, newTy: boolTy);
719 return builder.createLogicalOr(loc: op.getLoc(), lhs: srcRealToBool, rhs: srcImagToBool);
720}
721
722static mlir::Value lowerComplexToComplexCast(mlir::MLIRContext &ctx,
723 cir::CastOp op,
724 cir::CastKind scalarCastKind) {
725 CIRBaseBuilderTy builder(ctx);
726 builder.setInsertionPoint(op);
727
728 mlir::Value src = op.getSrc();
729 auto dstComplexElemTy =
730 mlir::cast<cir::ComplexType>(Val: op.getType()).getElementType();
731
732 mlir::Value srcReal = builder.createComplexReal(loc: op.getLoc(), operand: src);
733 mlir::Value srcImag = builder.createComplexImag(loc: op.getLoc(), operand: src);
734
735 mlir::Value dstReal = builder.createCast(loc: op.getLoc(), kind: scalarCastKind, src: srcReal,
736 newTy: dstComplexElemTy);
737 mlir::Value dstImag = builder.createCast(loc: op.getLoc(), kind: scalarCastKind, src: srcImag,
738 newTy: dstComplexElemTy);
739 return builder.createComplexCreate(loc: op.getLoc(), real: dstReal, imag: dstImag);
740}
741
742void LoweringPreparePass::lowerCastOp(cir::CastOp op) {
743 mlir::MLIRContext &ctx = getContext();
744 mlir::Value loweredValue = [&]() -> mlir::Value {
745 switch (op.getKind()) {
746 case cir::CastKind::float_to_complex:
747 case cir::CastKind::int_to_complex:
748 return lowerScalarToComplexCast(ctx, op);
749 case cir::CastKind::float_complex_to_real:
750 case cir::CastKind::int_complex_to_real:
751 return lowerComplexToScalarCast(ctx, op, elemToBoolKind: op.getKind());
752 case cir::CastKind::float_complex_to_bool:
753 return lowerComplexToScalarCast(ctx, op, elemToBoolKind: cir::CastKind::float_to_bool);
754 case cir::CastKind::int_complex_to_bool:
755 return lowerComplexToScalarCast(ctx, op, elemToBoolKind: cir::CastKind::int_to_bool);
756 case cir::CastKind::float_complex:
757 return lowerComplexToComplexCast(ctx, op, scalarCastKind: cir::CastKind::floating);
758 case cir::CastKind::float_complex_to_int_complex:
759 return lowerComplexToComplexCast(ctx, op, scalarCastKind: cir::CastKind::float_to_int);
760 case cir::CastKind::int_complex:
761 return lowerComplexToComplexCast(ctx, op, scalarCastKind: cir::CastKind::integral);
762 case cir::CastKind::int_complex_to_float_complex:
763 return lowerComplexToComplexCast(ctx, op, scalarCastKind: cir::CastKind::int_to_float);
764 default:
765 return nullptr;
766 }
767 }();
768
769 if (loweredValue) {
770 op.replaceAllUsesWith(newValue: loweredValue);
771 op.erase();
772 }
773}
774
775static mlir::Value buildComplexBinOpLibCall(
776 LoweringPreparePass &pass, CIRBaseBuilderTy &builder,
777 llvm::StringRef (*libFuncNameGetter)(llvm::APFloat::Semantics),
778 mlir::Location loc, cir::ComplexType ty, mlir::Value lhsReal,
779 mlir::Value lhsImag, mlir::Value rhsReal, mlir::Value rhsImag) {
780 cir::FPTypeInterface elementTy =
781 mlir::cast<cir::FPTypeInterface>(Val: ty.getElementType());
782
783 llvm::StringRef libFuncName = libFuncNameGetter(
784 llvm::APFloat::SemanticsToEnum(Sem: elementTy.getFloatSemantics()));
785 llvm::SmallVector<mlir::Type, 4> libFuncInputTypes(4, elementTy);
786
787 cir::FuncType libFuncTy = cir::FuncType::get(inputs: libFuncInputTypes, returnType: ty);
788
789 // Insert a declaration for the runtime function to be used in Complex
790 // multiplication and division when needed
791 cir::FuncOp libFunc;
792 {
793 mlir::OpBuilder::InsertionGuard ipGuard{builder};
794 builder.setInsertionPointToStart(pass.mlirModule.getBody());
795 libFunc = pass.buildRuntimeFunction(builder, name: libFuncName, loc, type: libFuncTy);
796 }
797
798 // TODO(cir): set the runtime calling convention to this call.
799 assert(!cir::MissingFeatures::opFuncCallingConv());
800 cir::CallOp call =
801 builder.createCallOp(loc, callee: libFunc, operands: {lhsReal, lhsImag, rhsReal, rhsImag});
802 return call.getResult();
803}
804
805static llvm::StringRef
806getComplexDivLibCallName(llvm::APFloat::Semantics semantics) {
807 switch (semantics) {
808 case llvm::APFloat::S_IEEEhalf:
809 return "__divhc3";
810 case llvm::APFloat::S_IEEEsingle:
811 return "__divsc3";
812 case llvm::APFloat::S_IEEEdouble:
813 return "__divdc3";
814 case llvm::APFloat::S_PPCDoubleDouble:
815 return "__divtc3";
816 case llvm::APFloat::S_x87DoubleExtended:
817 return "__divxc3";
818 case llvm::APFloat::S_IEEEquad:
819 return "__divtc3";
820 default:
821 llvm_unreachable("unsupported floating point type");
822 }
823}
824
825static mlir::Value
826buildAlgebraicComplexDiv(CIRBaseBuilderTy &builder, mlir::Location loc,
827 mlir::Value lhsReal, mlir::Value lhsImag,
828 mlir::Value rhsReal, mlir::Value rhsImag) {
829 // (a+bi) / (c+di) = ((ac+bd)/(cc+dd)) + ((bc-ad)/(cc+dd))i
830 mlir::Value &a = lhsReal;
831 mlir::Value &b = lhsImag;
832 mlir::Value &c = rhsReal;
833 mlir::Value &d = rhsImag;
834
835 // The element type of the complex (lhs/rhs) determines whether floating
836 // point or integer ops are needed.
837 bool isFP = cir::isFPOrVectorOfFPType(type: a.getType());
838 auto mul = [&](mlir::Location l, mlir::Value x, mlir::Value y) {
839 return isFP ? builder.createFMul(loc: l, lhs: x, rhs: y) : builder.createMul(loc: l, lhs: x, rhs: y);
840 };
841 auto add = [&](mlir::Location l, mlir::Value x, mlir::Value y) {
842 return isFP ? builder.createFAdd(loc: l, lhs: x, rhs: y) : builder.createAdd(loc: l, lhs: x, rhs: y);
843 };
844 auto sub = [&](mlir::Location l, mlir::Value x, mlir::Value y) {
845 return isFP ? builder.createFSub(loc: l, lhs: x, rhs: y) : builder.createSub(loc: l, lhs: x, rhs: y);
846 };
847 auto div = [&](mlir::Location l, mlir::Value x, mlir::Value y) {
848 return isFP ? builder.createFDiv(loc: l, lhs: x, rhs: y) : builder.createDiv(loc: l, lhs: x, rhs: y);
849 };
850
851 mlir::Value ac = mul(loc, a, c); // a*c
852 mlir::Value bd = mul(loc, b, d); // b*d
853 mlir::Value cc = mul(loc, c, c); // c*c
854 mlir::Value dd = mul(loc, d, d); // d*d
855 mlir::Value acbd = add(loc, ac, bd); // ac+bd
856 mlir::Value ccdd = add(loc, cc, dd); // cc+dd
857 mlir::Value resultReal = div(loc, acbd, ccdd);
858
859 mlir::Value bc = mul(loc, b, c); // b*c
860 mlir::Value ad = mul(loc, a, d); // a*d
861 mlir::Value bcad = sub(loc, bc, ad); // bc-ad
862 mlir::Value resultImag = div(loc, bcad, ccdd);
863 return builder.createComplexCreate(loc, real: resultReal, imag: resultImag);
864}
865
866static mlir::Value
867buildRangeReductionComplexDiv(CIRBaseBuilderTy &builder, mlir::Location loc,
868 mlir::Value lhsReal, mlir::Value lhsImag,
869 mlir::Value rhsReal, mlir::Value rhsImag) {
870 // Implements Smith's algorithm for complex division.
871 // SMITH, R. L. Algorithm 116: Complex division. Commun. ACM 5, 8 (1962).
872
873 // Let:
874 // - lhs := a+bi
875 // - rhs := c+di
876 // - result := lhs / rhs = e+fi
877 //
878 // The algorithm pseudocode looks like follows:
879 // if fabs(c) >= fabs(d):
880 // r := d / c
881 // tmp := c + r*d
882 // e = (a + b*r) / tmp
883 // f = (b - a*r) / tmp
884 // else:
885 // r := c / d
886 // tmp := d + r*c
887 // e = (a*r + b) / tmp
888 // f = (b*r - a) / tmp
889
890 mlir::Value &a = lhsReal;
891 mlir::Value &b = lhsImag;
892 mlir::Value &c = rhsReal;
893 mlir::Value &d = rhsImag;
894
895 // Smith's algorithm is only used for floating-point complex division.
896 assert(cir::isFPOrVectorOfFPType(a.getType()) &&
897 "range-reduction complex divide expects floating-point operands");
898
899 auto trueBranchBuilder = [&](mlir::OpBuilder &, mlir::Location) {
900 mlir::Value r = builder.createFDiv(loc, lhs: d, rhs: c); // r := d / c
901 mlir::Value rd = builder.createFMul(loc, lhs: r, rhs: d); // r*d
902 mlir::Value tmp = builder.createFAdd(loc, lhs: c, rhs: rd); // tmp := c + r*d
903
904 mlir::Value br = builder.createFMul(loc, lhs: b, rhs: r); // b*r
905 mlir::Value abr = builder.createFAdd(loc, lhs: a, rhs: br); // a + b*r
906 mlir::Value e = builder.createFDiv(loc, lhs: abr, rhs: tmp);
907
908 mlir::Value ar = builder.createFMul(loc, lhs: a, rhs: r); // a*r
909 mlir::Value bar = builder.createFSub(loc, lhs: b, rhs: ar); // b - a*r
910 mlir::Value f = builder.createFDiv(loc, lhs: bar, rhs: tmp);
911
912 mlir::Value result = builder.createComplexCreate(loc, real: e, imag: f);
913 builder.createYield(loc, value: result);
914 };
915
916 auto falseBranchBuilder = [&](mlir::OpBuilder &, mlir::Location) {
917 mlir::Value r = builder.createFDiv(loc, lhs: c, rhs: d); // r := c / d
918 mlir::Value rc = builder.createFMul(loc, lhs: r, rhs: c); // r*c
919 mlir::Value tmp = builder.createFAdd(loc, lhs: d, rhs: rc); // tmp := d + r*c
920
921 mlir::Value ar = builder.createFMul(loc, lhs: a, rhs: r); // a*r
922 mlir::Value arb = builder.createFAdd(loc, lhs: ar, rhs: b); // a*r + b
923 mlir::Value e = builder.createFDiv(loc, lhs: arb, rhs: tmp);
924
925 mlir::Value br = builder.createFMul(loc, lhs: b, rhs: r); // b*r
926 mlir::Value bra = builder.createFSub(loc, lhs: br, rhs: a); // b*r - a
927 mlir::Value f = builder.createFDiv(loc, lhs: bra, rhs: tmp);
928
929 mlir::Value result = builder.createComplexCreate(loc, real: e, imag: f);
930 builder.createYield(loc, value: result);
931 };
932
933 auto cFabs = cir::FAbsOp::create(builder, location: loc, src: c);
934 auto dFabs = cir::FAbsOp::create(builder, location: loc, src: d);
935 cir::CmpOp cmpResult =
936 builder.createCompare(loc, kind: cir::CmpOpKind::ge, lhs: cFabs, rhs: dFabs);
937 auto ternary = cir::TernaryOp::create(builder, location: loc, cond: cmpResult,
938 trueBuilder: trueBranchBuilder, falseBuilder: falseBranchBuilder);
939
940 return ternary.getResult();
941}
942
943static mlir::Type higherPrecisionElementTypeForComplexArithmetic(
944 mlir::MLIRContext &context, const clang::TargetInfo &targetInfo,
945 const clang::LangOptions &langOpts, CIRBaseBuilderTy &builder,
946 mlir::Type elementType) {
947
948 auto getHigherPrecisionFPType = [&context,
949 &targetInfo](mlir::Type type) -> mlir::Type {
950 if (mlir::isa<cir::FP16Type>(Val: type))
951 return cir::SingleType::get(ctx: &context);
952
953 if (mlir::isa<cir::SingleType>(Val: type) || mlir::isa<cir::BF16Type>(Val: type))
954 return cir::DoubleType::get(ctx: &context);
955
956 if (mlir::isa<cir::DoubleType>(Val: type)) {
957 const llvm::fltSemantics &format = targetInfo.getLongDoubleFormat();
958 mlir::Type underlying;
959 if (&format == &llvm::APFloat::IEEEdouble())
960 underlying = cir::DoubleType::get(ctx: &context);
961 else if (&format == &llvm::APFloat::x87DoubleExtended())
962 underlying = cir::FP80Type::get(ctx: &context);
963 else if (&format == &llvm::APFloat::IEEEquad())
964 underlying = cir::FP128Type::get(ctx: &context);
965 else
966 llvm_unreachable("NYI: unsupported long double format");
967 return cir::LongDoubleType::get(context: &context, underlying);
968 }
969
970 return type;
971 };
972
973 auto getFloatTypeSemantics =
974 [&langOpts, &targetInfo](mlir::Type type) -> const llvm::fltSemantics & {
975 const clang::TargetInfo &info = targetInfo;
976 if (mlir::isa<cir::FP16Type>(Val: type))
977 return info.getHalfFormat();
978
979 if (mlir::isa<cir::BF16Type>(Val: type))
980 return info.getBFloat16Format();
981
982 if (mlir::isa<cir::SingleType>(Val: type))
983 return info.getFloatFormat();
984
985 if (mlir::isa<cir::DoubleType>(Val: type))
986 return info.getDoubleFormat();
987
988 if (mlir::isa<cir::LongDoubleType>(Val: type)) {
989 if (langOpts.OpenMP && langOpts.OpenMPIsTargetDevice)
990 llvm_unreachable("NYI Float type semantics with OpenMP");
991 return info.getLongDoubleFormat();
992 }
993
994 if (mlir::isa<cir::FP128Type>(Val: type)) {
995 if (langOpts.OpenMP && langOpts.OpenMPIsTargetDevice)
996 llvm_unreachable("NYI Float type semantics with OpenMP");
997 return info.getFloat128Format();
998 }
999
1000 llvm_unreachable("Unsupported float type semantics");
1001 };
1002
1003 const mlir::Type higherElementType = getHigherPrecisionFPType(elementType);
1004 const llvm::fltSemantics &elementTypeSemantics =
1005 getFloatTypeSemantics(elementType);
1006 const llvm::fltSemantics &higherElementTypeSemantics =
1007 getFloatTypeSemantics(higherElementType);
1008
1009 // Check that the promoted type can handle the intermediate values without
1010 // overflowing. This can be interpreted as:
1011 // (SmallerType.LargestFiniteVal * SmallerType.LargestFiniteVal) * 2 <=
1012 // LargerType.LargestFiniteVal.
1013 // In terms of exponent it gives this formula:
1014 // (SmallerType.LargestFiniteVal * SmallerType.LargestFiniteVal
1015 // doubles the exponent of SmallerType.LargestFiniteVal)
1016 if (llvm::APFloat::semanticsMaxExponent(elementTypeSemantics) * 2 + 1 <=
1017 llvm::APFloat::semanticsMaxExponent(higherElementTypeSemantics)) {
1018 return higherElementType;
1019 }
1020
1021 // The intermediate values can't be represented in the promoted type
1022 // without overflowing.
1023 return {};
1024}
1025
1026static mlir::Value lowerComplexFDiv(LoweringPreparePass &pass,
1027 CIRBaseBuilderTy &builder,
1028 mlir::Location loc, cir::ComplexFDivOp op,
1029 mlir::Value lhsReal, mlir::Value lhsImag,
1030 mlir::Value rhsReal, mlir::Value rhsImag,
1031 mlir::MLIRContext &mlirCx,
1032 const clang::TargetInfo &targetInfo) {
1033 cir::ComplexType complexTy = op.getType();
1034 cir::ComplexRangeKind range = op.getRange();
1035 switch (range) {
1036 case ComplexRangeKind::Full: {
1037 return buildComplexBinOpLibCall(pass, builder, libFuncNameGetter: &getComplexDivLibCallName,
1038 loc, ty: complexTy, lhsReal, lhsImag, rhsReal,
1039 rhsImag);
1040 }
1041 case ComplexRangeKind::Improved: {
1042 return buildRangeReductionComplexDiv(builder, loc, lhsReal, lhsImag,
1043 rhsReal, rhsImag);
1044 }
1045 case ComplexRangeKind::Promoted: {
1046 mlir::Type originalElementType = complexTy.getElementType();
1047 mlir::Type higherPrecisionElementType =
1048 higherPrecisionElementTypeForComplexArithmetic(
1049 context&: mlirCx, targetInfo, langOpts: pass.getLangOpts(), builder,
1050 elementType: originalElementType);
1051
1052 if (!higherPrecisionElementType)
1053 return buildRangeReductionComplexDiv(builder, loc, lhsReal, lhsImag,
1054 rhsReal, rhsImag);
1055
1056 cir::CastKind floatingCastKind = cir::CastKind::floating;
1057 lhsReal = builder.createCast(kind: floatingCastKind, src: lhsReal,
1058 newTy: higherPrecisionElementType);
1059 lhsImag = builder.createCast(kind: floatingCastKind, src: lhsImag,
1060 newTy: higherPrecisionElementType);
1061 rhsReal = builder.createCast(kind: floatingCastKind, src: rhsReal,
1062 newTy: higherPrecisionElementType);
1063 rhsImag = builder.createCast(kind: floatingCastKind, src: rhsImag,
1064 newTy: higherPrecisionElementType);
1065
1066 mlir::Value algebraicResult = buildAlgebraicComplexDiv(
1067 builder, loc, lhsReal, lhsImag, rhsReal, rhsImag);
1068
1069 mlir::Value resultReal = builder.createComplexReal(loc, operand: algebraicResult);
1070 mlir::Value resultImag = builder.createComplexImag(loc, operand: algebraicResult);
1071
1072 mlir::Value finalReal =
1073 builder.createCast(kind: floatingCastKind, src: resultReal, newTy: originalElementType);
1074 mlir::Value finalImag =
1075 builder.createCast(kind: floatingCastKind, src: resultImag, newTy: originalElementType);
1076 return builder.createComplexCreate(loc, real: finalReal, imag: finalImag);
1077 }
1078 case ComplexRangeKind::Basic: {
1079 return buildAlgebraicComplexDiv(builder, loc, lhsReal, lhsImag, rhsReal,
1080 rhsImag);
1081 }
1082 }
1083 llvm_unreachable("fully covered switch");
1084}
1085
1086void LoweringPreparePass::lowerComplexFDivOp(cir::ComplexFDivOp op) {
1087 cir::CIRBaseBuilderTy builder(getContext());
1088 builder.setInsertionPointAfter(op);
1089 mlir::Location loc = op.getLoc();
1090 mlir::TypedValue<cir::ComplexType> lhs = op.getLhs();
1091 mlir::TypedValue<cir::ComplexType> rhs = op.getRhs();
1092 mlir::Value lhsReal = builder.createComplexReal(loc, operand: lhs);
1093 mlir::Value lhsImag = builder.createComplexImag(loc, operand: lhs);
1094 mlir::Value rhsReal = builder.createComplexReal(loc, operand: rhs);
1095 mlir::Value rhsImag = builder.createComplexImag(loc, operand: rhs);
1096
1097 mlir::Value loweredResult =
1098 lowerComplexFDiv(pass&: *this, builder, loc, op, lhsReal, lhsImag, rhsReal,
1099 rhsImag, mlirCx&: getContext(), targetInfo: getTargetInfo());
1100 op.replaceAllUsesWith(newValue: loweredResult);
1101 op.erase();
1102}
1103
1104void LoweringPreparePass::lowerComplexDivOp(cir::ComplexDivOp op) {
1105 cir::CIRBaseBuilderTy builder(getContext());
1106 builder.setInsertionPointAfter(op);
1107 mlir::Location loc = op.getLoc();
1108 mlir::TypedValue<cir::ComplexType> lhs = op.getLhs();
1109 mlir::TypedValue<cir::ComplexType> rhs = op.getRhs();
1110 mlir::Value lhsReal = builder.createComplexReal(loc, operand: lhs);
1111 mlir::Value lhsImag = builder.createComplexImag(loc, operand: lhs);
1112 mlir::Value rhsReal = builder.createComplexReal(loc, operand: rhs);
1113 mlir::Value rhsImag = builder.createComplexImag(loc, operand: rhs);
1114
1115 mlir::Value loweredResult = buildAlgebraicComplexDiv(
1116 builder, loc, lhsReal, lhsImag, rhsReal, rhsImag);
1117 op.replaceAllUsesWith(newValue: loweredResult);
1118 op.erase();
1119}
1120
1121static llvm::StringRef
1122getComplexMulLibCallName(llvm::APFloat::Semantics semantics) {
1123 switch (semantics) {
1124 case llvm::APFloat::S_IEEEhalf:
1125 return "__mulhc3";
1126 case llvm::APFloat::S_IEEEsingle:
1127 return "__mulsc3";
1128 case llvm::APFloat::S_IEEEdouble:
1129 return "__muldc3";
1130 case llvm::APFloat::S_PPCDoubleDouble:
1131 return "__multc3";
1132 case llvm::APFloat::S_x87DoubleExtended:
1133 return "__mulxc3";
1134 case llvm::APFloat::S_IEEEquad:
1135 return "__multc3";
1136 default:
1137 llvm_unreachable("unsupported floating point type");
1138 }
1139}
1140
1141static mlir::Value lowerComplexFMul(LoweringPreparePass &pass,
1142 CIRBaseBuilderTy &builder,
1143 mlir::Location loc, cir::ComplexFMulOp op,
1144 mlir::Value lhsReal, mlir::Value lhsImag,
1145 mlir::Value rhsReal, mlir::Value rhsImag) {
1146 // (a+bi) * (c+di) = (ac-bd) + (ad+bc)i
1147 mlir::Value resultRealLhs = builder.createFMul(loc, lhs: lhsReal, rhs: rhsReal); // ac
1148 mlir::Value resultRealRhs = builder.createFMul(loc, lhs: lhsImag, rhs: rhsImag); // bd
1149 mlir::Value resultImagLhs = builder.createFMul(loc, lhs: lhsReal, rhs: rhsImag); // ad
1150 mlir::Value resultImagRhs = builder.createFMul(loc, lhs: lhsImag, rhs: rhsReal); // bc
1151 mlir::Value resultReal =
1152 builder.createFSub(loc, lhs: resultRealLhs, rhs: resultRealRhs);
1153 mlir::Value resultImag =
1154 builder.createFAdd(loc, lhs: resultImagLhs, rhs: resultImagRhs);
1155 mlir::Value algebraicResult =
1156 builder.createComplexCreate(loc, real: resultReal, imag: resultImag);
1157
1158 cir::ComplexType complexTy = op.getType();
1159 cir::ComplexRangeKind rangeKind = op.getRange();
1160 if (rangeKind == cir::ComplexRangeKind::Basic ||
1161 rangeKind == cir::ComplexRangeKind::Improved ||
1162 rangeKind == cir::ComplexRangeKind::Promoted)
1163 return algebraicResult;
1164
1165 assert(!cir::MissingFeatures::fastMathFlags());
1166
1167 // Check whether the real part and the imaginary part of the result are both
1168 // NaN. If so, emit a library call to compute the multiplication instead.
1169 // We check a value against NaN by comparing the value against itself.
1170 mlir::Value resultRealIsNaN = builder.createIsNaN(loc, operand: resultReal);
1171 mlir::Value resultImagIsNaN = builder.createIsNaN(loc, operand: resultImag);
1172 mlir::Value resultRealAndImagAreNaN =
1173 builder.createLogicalAnd(loc, lhs: resultRealIsNaN, rhs: resultImagIsNaN);
1174
1175 return cir::TernaryOp::create(
1176 builder, location: loc, cond: resultRealAndImagAreNaN,
1177 trueBuilder: [&](mlir::OpBuilder &, mlir::Location) {
1178 mlir::Value libCallResult = buildComplexBinOpLibCall(
1179 pass, builder, libFuncNameGetter: &getComplexMulLibCallName, loc, ty: complexTy,
1180 lhsReal, lhsImag, rhsReal, rhsImag);
1181 builder.createYield(loc, value: libCallResult);
1182 },
1183 falseBuilder: [&](mlir::OpBuilder &, mlir::Location) {
1184 builder.createYield(loc, value: algebraicResult);
1185 })
1186 .getResult();
1187}
1188
1189void LoweringPreparePass::lowerComplexFMulOp(cir::ComplexFMulOp op) {
1190 cir::CIRBaseBuilderTy builder(getContext());
1191 builder.setInsertionPointAfter(op);
1192 mlir::Location loc = op.getLoc();
1193 mlir::TypedValue<cir::ComplexType> lhs = op.getLhs();
1194 mlir::TypedValue<cir::ComplexType> rhs = op.getRhs();
1195
1196 // (a+bi) * (c+di) = (ac-bd) + (ad+bc)i
1197 mlir::Value lhsReal = builder.createComplexReal(loc, operand: lhs);
1198 mlir::Value lhsImag = builder.createComplexImag(loc, operand: lhs);
1199 mlir::Value rhsReal = builder.createComplexReal(loc, operand: rhs);
1200 mlir::Value rhsImag = builder.createComplexImag(loc, operand: rhs);
1201 mlir::Value loweredResult = lowerComplexFMul(pass&: *this, builder, loc, op, lhsReal,
1202 lhsImag, rhsReal, rhsImag);
1203 op.replaceAllUsesWith(newValue: loweredResult);
1204 op.erase();
1205}
1206
1207void LoweringPreparePass::lowerComplexMulOp(cir::ComplexMulOp op) {
1208 cir::CIRBaseBuilderTy builder(getContext());
1209 builder.setInsertionPointAfter(op);
1210 mlir::Location loc = op.getLoc();
1211 mlir::TypedValue<cir::ComplexType> lhs = op.getLhs();
1212 mlir::TypedValue<cir::ComplexType> rhs = op.getRhs();
1213 mlir::Value lhsReal = builder.createComplexReal(loc, operand: lhs);
1214 mlir::Value lhsImag = builder.createComplexImag(loc, operand: lhs);
1215 mlir::Value rhsReal = builder.createComplexReal(loc, operand: rhs);
1216 mlir::Value rhsImag = builder.createComplexImag(loc, operand: rhs);
1217
1218 mlir::Value resultRealLhs = builder.createMul(loc, lhs: lhsReal, rhs: rhsReal); // ac
1219 mlir::Value resultRealRhs = builder.createMul(loc, lhs: lhsImag, rhs: rhsImag); // bd
1220 mlir::Value resultImagLhs = builder.createMul(loc, lhs: lhsReal, rhs: rhsImag); // ad
1221 mlir::Value resultImagRhs = builder.createMul(loc, lhs: lhsImag, rhs: rhsReal); // bc
1222 mlir::Value resultReal = builder.createSub(loc, lhs: resultRealLhs, rhs: resultRealRhs);
1223 mlir::Value resultImag = builder.createAdd(loc, lhs: resultImagLhs, rhs: resultImagRhs);
1224
1225 mlir::Value loweredResult =
1226 builder.createComplexCreate(loc, real: resultReal, imag: resultImag);
1227
1228 op.replaceAllUsesWith(newValue: loweredResult);
1229 op.erase();
1230}
1231
1232void LoweringPreparePass::lowerComplexConjOp(cir::ComplexConjOp op) {
1233 mlir::Location loc = op.getLoc();
1234 CIRBaseBuilderTy builder(getContext());
1235 builder.setInsertionPointAfter(op);
1236
1237 mlir::Value operand = op.getOperand();
1238 mlir::Value operandReal = builder.createComplexReal(loc, operand);
1239 mlir::Value operandImag = builder.createComplexImag(loc, operand);
1240
1241 // The complex conjugate is formed by negating the imaginary component.
1242 const bool isFP = cir::isFPOrVectorOfFPType(type: operandReal.getType());
1243 mlir::Value resultImag = isFP ? builder.createFNeg(loc, operand: operandImag)
1244 : builder.createMinus(loc, input: operandImag);
1245
1246 mlir::Value result =
1247 builder.createComplexCreate(loc, real: operandReal, imag: resultImag);
1248 op->replaceAllUsesWith(values: mlir::ValueRange{result});
1249 op->erase();
1250}
1251
1252cir::FuncOp
1253LoweringPreparePass::buildGlobalArrayDtorFunc(CIRBaseBuilderTy &builder,
1254 mlir::Operation *op) {
1255 mlir::OpBuilder::InsertionGuard guard(builder);
1256 builder.setInsertionPointAfter(op);
1257 SmallString<256> fnName("__cxx_global_array_dtor");
1258 uint32_t cnt = dynamicInitializerNames[fnName]++;
1259 if (cnt)
1260 fnName += "." + std::to_string(val: cnt);
1261
1262 auto fnType =
1263 cir::FuncType::get(inputs: {builder.getVoidPtrTy()}, returnType: builder.getVoidTy());
1264 cir::FuncOp dtorFunc =
1265 buildRuntimeFunction(builder, name: fnName, loc: op->getLoc(), type: fnType,
1266 linkage: cir::GlobalLinkageKind::InternalLinkage,
1267 /*hasDefaultFuncAttrs=*/true);
1268
1269 SmallVector<mlir::NamedAttribute> paramAttrs;
1270 paramAttrs.push_back(
1271 Elt: builder.getNamedAttr(name: "llvm.noundef", val: builder.getUnitAttr()));
1272 SmallVector<mlir::Attribute> argAttrDicts;
1273 argAttrDicts.push_back(
1274 Elt: mlir::DictionaryAttr::get(context: builder.getContext(), value: paramAttrs));
1275 dtorFunc.setArgAttrsAttr(
1276 mlir::ArrayAttr::get(context: builder.getContext(), value: argAttrDicts));
1277
1278 dtorFunc.addEntryBlock();
1279 return dtorFunc;
1280}
1281
1282void LoweringPreparePass::lowerRegisterExitDtorOp(cir::RegisterExitDtorOp op) {
1283 CIRBaseBuilderTy builder(getContext());
1284 mlir::Location loc = op.getLoc();
1285 auto object = symbolTables.lookupNearestSymbolFrom<cir::GlobalOp>(
1286 from: op, symbol: op.getObjectAttr());
1287 assert(object && "register_exit_dtor must reference a cir.global");
1288 mlir::Block &body = op.getBody().front();
1289 builder.setInsertionPoint(op);
1290
1291 // When the body is a single call on the object's address, register the
1292 // callee directly with that address.
1293 cir::FuncOp dtorFunc;
1294 mlir::Value arg;
1295 if (body.getOperations().size() == 3) {
1296 auto getGlobal = mlir::dyn_cast<cir::GetGlobalOp>(Val&: body.front());
1297 auto call = mlir::dyn_cast<cir::CallOp>(Val&: *std::next(x: body.begin()));
1298 if (getGlobal && call && getGlobal.getName() == op.getObject() &&
1299 call.getNumArgOperands() == 1 &&
1300 call.getArgOperand(i: 0) == getGlobal.getResult()) {
1301 dtorFunc = getCalledFunction(callOp: call);
1302 if (dtorFunc)
1303 arg = builder.clone(op&: *getGlobal)->getResult(idx: 0);
1304 }
1305 }
1306
1307 // Otherwise the body moves into a helper that ignores its argument, and the
1308 // helper is registered with a null argument.
1309 if (!dtorFunc) {
1310 dtorFunc = buildGlobalArrayDtorFunc(builder, op: object);
1311 mlir::Block *entryBB = &dtorFunc.getBody().front();
1312 entryBB->getOperations().splice(where: entryBB->end(), L2&: body.getOperations());
1313 auto yieldOp = mlir::cast<cir::YieldOp>(Val: entryBB->getTerminator());
1314 {
1315 mlir::OpBuilder::InsertionGuard guard(builder);
1316 builder.setInsertionPoint(yieldOp);
1317 cir::ReturnOp::create(builder, location: yieldOp.getLoc());
1318 }
1319 yieldOp->erase();
1320 arg = builder.getNullPtr(ty: builder.getVoidPtrTy(), loc);
1321 }
1322
1323 cir::CallOp atExitCall = emitAtExitCall(builder, loc, dtorFunc, arg,
1324 tls: object.getTlsModel().has_value());
1325 // The atexit functions do not throw, and marking the call so keeps it out of
1326 // an invoke inside the initializer.
1327 atExitCall.setNothrowAttr(builder.getUnitAttr());
1328 op->erase();
1329}
1330
1331cir::FuncOp LoweringPreparePass::getOrCreateDtorFunc(CIRBaseBuilderTy &builder,
1332 cir::GlobalOp op,
1333 mlir::Region &dtorRegion,
1334 cir::CallOp &dtorCall) {
1335 mlir::OpBuilder::InsertionGuard guard(builder);
1336 assert(!cir::MissingFeatures::astVarDeclInterface());
1337
1338 // Look for operations in dtorBlock
1339 mlir::Block &dtorBlock = dtorRegion.front();
1340
1341 // The first operation should be a get_global to retrieve the address
1342 // of the global variable we're destroying.
1343 auto opIt = dtorBlock.getOperations().begin();
1344 cir::GetGlobalOp ggop = mlir::cast<cir::GetGlobalOp>(Val&: *opIt);
1345
1346 // The simple case is just a call to a destructor, like this:
1347 //
1348 // %0 = cir.get_global %globalS : !cir.ptr<!rec_S>
1349 // cir.call %_ZN1SD1Ev(%0) : (!cir.ptr<!rec_S>) -> ()
1350 // (implicit cir.yield)
1351 //
1352 // That is, if the second operation is a call that takes the get_global
1353 // result as its only operand, and the only other operation is a yield, then
1354 // we can just return the called function.
1355 if (dtorBlock.getOperations().size() == 3) {
1356 auto callOp = mlir::dyn_cast<cir::CallOp>(Val: &*(++opIt));
1357 auto yieldOp = mlir::dyn_cast<cir::YieldOp>(Val: &*(++opIt));
1358 if (yieldOp && callOp && callOp.getNumOperands() == 1 &&
1359 callOp.getArgOperand(i: 0) == ggop) {
1360 dtorCall = callOp;
1361 return getCalledFunction(callOp);
1362 }
1363 }
1364
1365 // Otherwise, we need to create a helper function to replace the dtor region.
1366 cir::FuncOp dtorFunc = buildGlobalArrayDtorFunc(builder, op);
1367 mlir::Block *entryBB = &dtorFunc.getBody().front();
1368
1369 // Move everything from the dtor region into the helper function.
1370 entryBB->getOperations().splice(where: entryBB->begin(), L2&: dtorBlock.getOperations(),
1371 first: dtorBlock.begin(), last: dtorBlock.end());
1372
1373 // Before erasing this, clone it back into the dtor region
1374 cir::GetGlobalOp dtorGGop =
1375 mlir::cast<cir::GetGlobalOp>(Val&: entryBB->getOperations().front());
1376 builder.setInsertionPointToStart(&dtorBlock);
1377 builder.clone(op&: *dtorGGop.getOperation());
1378
1379 // Replace all uses of the help function's get_global with the function
1380 // argument.
1381 mlir::Value dtorArg = entryBB->getArgument(i: 0);
1382 // The arg is a void*, but the destructor needs the variables type, so we
1383 // might need to introduce a cast.
1384 if (dtorArg.getType() != dtorGGop.getType()) {
1385 builder.setInsertionPointToStart(entryBB);
1386 dtorArg =
1387 builder.createBitcast(loc: dtorGGop.getLoc(), src: dtorArg, newTy: dtorGGop.getType());
1388 }
1389 dtorGGop.replaceAllUsesWith(newValue: dtorArg);
1390 dtorGGop.erase();
1391
1392 // Replace the yield in the final block with a return
1393 mlir::Block &finalBlock = dtorFunc.getBody().back();
1394 auto yieldOp = cast<cir::YieldOp>(Val: finalBlock.getTerminator());
1395 builder.setInsertionPoint(yieldOp);
1396 cir::ReturnOp::create(builder, location: yieldOp->getLoc());
1397 yieldOp->erase();
1398
1399 // Create a call to the helper function, passing the original get_global op
1400 // as the argument.
1401 cir::GetGlobalOp origGGop =
1402 mlir::cast<cir::GetGlobalOp>(Val&: dtorBlock.getOperations().front());
1403 builder.setInsertionPointAfter(origGGop);
1404 mlir::Value ggopResult = origGGop.getResult();
1405 dtorCall = builder.createCallOp(loc: op.getLoc(), callee: dtorFunc, operands: ggopResult,
1406 /*attrs=*/{}, /*argAttrs=*/{},
1407 /*resAttrs=*/{}, callingConv: dtorFunc.getCallingConv());
1408
1409 // Add a yield after the call.
1410 auto finalYield = cir::YieldOp::create(builder, location: op.getLoc());
1411
1412 // Erase everything after the yield.
1413 dtorBlock.getOperations().erase(first: std::next(x: mlir::Block::iterator(finalYield)),
1414 last: dtorBlock.end());
1415 dtorRegion.getBlocks().erase(first: std::next(x: dtorRegion.begin()), last: dtorRegion.end());
1416
1417 return dtorFunc;
1418}
1419
1420cir::FuncOp
1421LoweringPreparePass::buildCXXGlobalVarDeclInitFunc(cir::GlobalOp op) {
1422 // TODO(cir): Store this in the GlobalOp.
1423 // This should come from the MangleContext, but for now I'm hardcoding it.
1424 SmallString<256> fnName("__cxx_global_var_init");
1425 // Get a unique name
1426 uint32_t cnt = dynamicInitializerNames[fnName]++;
1427 if (cnt)
1428 fnName += "." + std::to_string(val: cnt);
1429
1430 // Create a variable initialization function.
1431 CIRBaseBuilderTy builder(getContext());
1432 builder.setInsertionPointAfter(op);
1433 cir::VoidType voidTy = builder.getVoidTy();
1434 auto fnType = cir::FuncType::get(inputs: {}, returnType: voidTy);
1435 FuncOp f = buildRuntimeFunction(builder, name: fnName, loc: op.getLoc(), type: fnType,
1436 linkage: cir::GlobalLinkageKind::InternalLinkage,
1437 /*hasDefaultFuncAttrs=*/true);
1438
1439 // Forward the constrained floating-point marker recorded on the global by
1440 // CodeGen onto the generated initializer function. The marker on the global
1441 // is no longer meaningful once its regions have been moved out, so clear it.
1442 if (op.getStrictfp()) {
1443 f->setAttr(name: cir::CIRDialect::getStrictFPAttrName(),
1444 value: mlir::UnitAttr::get(context: &getContext()));
1445 op.setStrictfp(false);
1446 }
1447
1448 // Move over the initialization code of the ctor region.
1449 // The ctor region may have multiple blocks when exception handling
1450 // scaffolding creates extra blocks (e.g., unreachable/trap blocks).
1451 // We move all operations from the first block (minus the yield) into
1452 // the function entry, and discard extra blocks (which contain only
1453 // unreachable terminators from EH cleanup paths).
1454 mlir::Block *entryBB = f.addEntryBlock();
1455 builder.setInsertionPointToStart(entryBB);
1456
1457 // If this is a global TLS variable (that is, declared at namespace scope), we
1458 // have to emit the guard variable here.
1459 bool needsTlsGuard = op.getTlsRefs() && op.getTlsRefs()->getGuardName();
1460
1461 // Non-TLS globals (such as weak/linkonce_odr/inline-ns scope variables) use
1462 // the __cxa_guard_acquire/release instead of the TLS guard.
1463 if (!needsTlsGuard && op.getDynamicInitGuard()) {
1464 if (!op.getDynamicInitInfo()) {
1465 op->emitError(
1466 message: "guarded global with a guard is missing 'dynamic_init_info'");
1467 return {};
1468 }
1469
1470 if (op.getDynamicInitInfo()->getLocal()) {
1471 op->emitError(message: "guarded global is marked local");
1472 return {};
1473 }
1474 emitGuardedInit(builder, globalOp: op, ctorRegion&: op.getCtorRegion(), dtorRegion&: op.getDtorRegion());
1475 builder.setInsertionPointToEnd(entryBB);
1476 cir::ReturnOp::create(builder, location: op.getLoc());
1477 return f;
1478 }
1479
1480 cir::IfOp guardIf;
1481 if (needsTlsGuard) {
1482 guardIf = buildInitGuardCheck(
1483 builder, loc: op.getLoc(),
1484 guard: getOrCreateStaticLocalDeclGuardAddress(
1485 builder, globalOp: op, guardName: op.getTlsRefs()->getGuardName().getValue(),
1486 /*isLocalVarDecl=*/false,
1487 /*useInt8GuardVariable=*/op.hasInternalLinkage()),
1488 /*threadLocal=*/true);
1489 builder.setInsertionPointToEnd(&guardIf.getThenRegion().front());
1490 }
1491
1492 if (!op.getCtorRegion().empty()) {
1493 mlir::Block &block = op.getCtorRegion().front();
1494 mlir::Block *insertBlock = builder.getBlock();
1495 insertBlock->getOperations().splice(where: insertBlock->end(),
1496 L2&: block.getOperations(), first: block.begin(),
1497 last: std::prev(x: block.end()));
1498 }
1499
1500 // Register the destructor call with __cxa_atexit
1501 mlir::Region &dtorRegion = op.getDtorRegion();
1502 if (!dtorRegion.empty()) {
1503 assert(!cir::MissingFeatures::astVarDeclInterface());
1504
1505 emitGlobalGuardedDtorRegion(builder, global: op, dtorRegion,
1506 tls: op.getTlsModel().has_value(),
1507 entryBB&: *builder.getBlock());
1508 }
1509
1510 // If we're actually in the 'if' above, create a yield.
1511 if (needsTlsGuard) {
1512 builder.setInsertionPointToEnd(&guardIf.getThenRegion().back());
1513 cir::YieldOp::create(builder, location: op.getLoc());
1514 }
1515
1516 // Replace cir.yield with cir.return
1517 builder.setInsertionPointToEnd(entryBB);
1518 mlir::Operation *yieldOp = nullptr;
1519 if (!op.getCtorRegion().empty()) {
1520 mlir::Block &block = op.getCtorRegion().front();
1521 yieldOp = &block.getOperations().back();
1522 } else {
1523 assert(!dtorRegion.empty());
1524 mlir::Block &block = dtorRegion.front();
1525 yieldOp = &block.getOperations().back();
1526 }
1527
1528 assert(isa<cir::YieldOp>(*yieldOp));
1529 cir::ReturnOp::create(builder, location: yieldOp->getLoc());
1530 return f;
1531}
1532
1533cir::FuncOp
1534LoweringPreparePass::getGuardAcquireFn(cir::PointerType guardPtrTy) {
1535 // int __cxa_guard_acquire(__guard *guard_object);
1536 CIRBaseBuilderTy builder(getContext());
1537 mlir::OpBuilder::InsertionGuard ipGuard{builder};
1538 builder.setInsertionPointToStart(mlirModule.getBody());
1539 mlir::Location loc = mlirModule.getLoc();
1540 cir::IntType intTy = cir::IntType::get(context: &getContext(), width: 32, /*isSigned=*/is_signed: true);
1541 auto fnType = cir::FuncType::get(inputs: {guardPtrTy}, returnType: intTy);
1542 return buildRuntimeFunction(builder, name: "__cxa_guard_acquire", loc, type: fnType);
1543}
1544
1545cir::FuncOp
1546LoweringPreparePass::getGuardReleaseFn(cir::PointerType guardPtrTy) {
1547 // void __cxa_guard_release(__guard *guard_object);
1548 CIRBaseBuilderTy builder(getContext());
1549 mlir::OpBuilder::InsertionGuard ipGuard{builder};
1550 builder.setInsertionPointToStart(mlirModule.getBody());
1551 mlir::Location loc = mlirModule.getLoc();
1552 cir::VoidType voidTy = cir::VoidType::get(ctx: &getContext());
1553 auto fnType = cir::FuncType::get(inputs: {guardPtrTy}, returnType: voidTy);
1554 return buildRuntimeFunction(builder, name: "__cxa_guard_release", loc, type: fnType);
1555}
1556
1557cir::FuncOp LoweringPreparePass::getGuardAbortFn(cir::PointerType guardPtrTy) {
1558 // void __cxa_guard_abort(__guard *guard_object);
1559 CIRBaseBuilderTy builder(getContext());
1560 mlir::OpBuilder::InsertionGuard ipGuard{builder};
1561 builder.setInsertionPointToStart(mlirModule.getBody());
1562 mlir::Location loc = mlirModule.getLoc();
1563 cir::VoidType voidTy = cir::VoidType::get(ctx: &getContext());
1564 auto fnType = cir::FuncType::get(inputs: {guardPtrTy}, returnType: voidTy);
1565 return buildRuntimeFunction(builder, name: "__cxa_guard_abort", loc, type: fnType);
1566}
1567
1568cir::FuncOp LoweringPreparePass::getTlsInitFn() {
1569 // void __tls_init(void);
1570 CIRBaseBuilderTy builder(getContext());
1571 mlir::OpBuilder::InsertionGuard _{builder};
1572 builder.setInsertionPointToStart(mlirModule.getBody());
1573 mlir::Location loc = mlirModule.getLoc();
1574 auto fnType = builder.getVoidFnTy();
1575 return buildRuntimeFunction(builder, name: "__tls_init", loc, type: fnType,
1576 linkage: cir::GlobalLinkageKind::InternalLinkage,
1577 /*hasDefaultFuncAttrs=*/true);
1578}
1579
1580cir::GlobalOp LoweringPreparePass::createGuardGlobalOp(
1581 CIRBaseBuilderTy &builder, mlir::Location loc, llvm::StringRef name,
1582 cir::IntType guardTy, cir::GlobalLinkageKind linkage) {
1583 mlir::OpBuilder::InsertionGuard guard(builder);
1584 builder.setInsertionPointToStart(mlirModule.getBody());
1585 cir::GlobalOp g = cir::GlobalOp::create(builder, location: loc, sym_name: name, sym_type: guardTy);
1586 g.setLinkageAttr(
1587 cir::GlobalLinkageKindAttr::get(context: builder.getContext(), value: linkage));
1588 mlir::SymbolTable::setSymbolVisibility(
1589 symbol: g, vis: mlir::SymbolTable::Visibility::Private);
1590 return g;
1591}
1592
1593void LoweringPreparePass::addUsedGlobal(cir::GlobalOp global) {
1594 CIRBaseBuilderTy builder(getContext());
1595 mlir::ptr::MemorySpaceAttrInterface eltAS;
1596 if (unsigned targetAS =
1597 getTargetInfo().getTargetAddressSpace(AS: clang::LangAS::Default))
1598 eltAS = cir::TargetAddressSpaceAttr::get(context: &getContext(), value: targetAS);
1599 cir::PointerType voidPtrTy = builder.getVoidPtrTy(as: eltAS);
1600 mlir::Attribute entry = cir::GlobalViewAttr::get(
1601 type: voidPtrTy, symbol: mlir::FlatSymbolRefAttr::get(value: global.getSymNameAttr()));
1602
1603 mlir::StringAttr usedName = builder.getStringAttr(bytes: "llvm.used");
1604 cir::GlobalOp used =
1605 symbolTables.lookupSymbolIn<cir::GlobalOp>(symbolTableOp: mlirModule, name&: usedName);
1606
1607 llvm::SmallVector<mlir::Attribute, 4> elements;
1608 if (used) {
1609 auto existing = mlir::cast<cir::ConstArrayAttr>(Val: *used.getInitialValue());
1610 llvm::append_range(C&: elements,
1611 R: mlir::cast<mlir::ArrayAttr>(Val: existing.getElts()));
1612 }
1613 elements.push_back(Elt: entry);
1614
1615 cir::ArrayType arrayTy = cir::ArrayType::get(elementType: voidPtrTy, size: elements.size());
1616 cir::ConstArrayAttr initAttr = cir::ConstArrayAttr::get(
1617 type: arrayTy, elts: mlir::ArrayAttr::get(context: &getContext(), value: elements));
1618
1619 if (used) {
1620 used.setSymType(arrayTy);
1621 used.setInitialValueAttr(initAttr);
1622 return;
1623 }
1624
1625 builder.setInsertionPointToStart(mlirModule.getBody());
1626 used = cir::GlobalOp::create(
1627 builder, location: mlirModule.getLoc(), sym_name: usedName.getValue(), sym_type: arrayTy,
1628 /*isConstant=*/false,
1629 addrSpace: cir::CIRDataLayout(mlirModule).getDefaultGlobalsAddrSpace(ctx: &getContext()));
1630 used.setLinkage(cir::GlobalLinkageKind::AppendingLinkage);
1631 used.setInitialValueAttr(initAttr);
1632 used.setSectionAttr(builder.getStringAttr(bytes: "llvm.metadata"));
1633 symbolTables.getSymbolTable(op: mlirModule).insert(symbol: used);
1634}
1635
1636void LoweringPreparePass::handleStaticLocal(cir::GlobalOp globalOp,
1637 cir::LocalInitOp localInitOp) {
1638 CIRBaseBuilderTy builder(getContext());
1639
1640 builder.setInsertionPointAfter(localInitOp);
1641 mlir::Block *localInitBlock = builder.getInsertionBlock();
1642
1643 // Remove the terminator temporarily - we'll add it back at the end.
1644 mlir::Operation *ret = localInitBlock->getTerminator();
1645 ret->remove();
1646 // Note: These two insert-point-after sets are necessary, as the 'trailing'
1647 // operation has changed thanks to the terminator removal.
1648 builder.setInsertionPointAfter(localInitOp);
1649
1650 emitGuardedInit(builder, globalOp, ctorRegion&: localInitOp.getCtorRegion(),
1651 dtorRegion&: localInitOp.getDtorRegion());
1652
1653 // Insert the removed terminator back.
1654 builder.getInsertionBlock()->push_back(op: ret);
1655}
1656
1657void LoweringPreparePass::emitGuardedInit(CIRBaseBuilderTy &builder,
1658 cir::GlobalOp globalOp,
1659 mlir::Region &ctorRegion,
1660 mlir::Region &dtorRegion) {
1661 // Dynamic-initialization facts are materialized into a serializable
1662 // attribute by CIRGen, so this pass does not need a live ASTContext to
1663 // read them.
1664 std::optional<cir::DynamicInitInfoAttr> infoOption =
1665 globalOp.getDynamicInitInfo();
1666
1667 // The GlobalOp verifier requires 'dynamic_init_info' whenever a
1668 // 'dynamic_init_guard' is present, so verified IR reaching here has it.
1669 // Emit an error instead of asserting so malformed .cir gets a diagnostic.
1670 if (!infoOption.has_value()) {
1671 globalOp->emitError(
1672 message: "dynamic-init-guarded global is missing 'dynamic_init_info'");
1673 return;
1674 }
1675
1676 cir::DynamicInitInfoAttr info = infoOption.value();
1677
1678 // Inline variables that weren't instantiated from variable templates have
1679 // partially-ordered initialization within their translation unit.
1680 bool nonTemplateInline =
1681 info.getIsInline() && !info.isTemplateInstantiation();
1682
1683 // We only need to use thread-safe statics for local non-TLS variables and
1684 // inline variables; other global initialization is always single-threaded
1685 // or (through lazy dynamic loading in multiple threads) unsequenced.
1686 bool threadsafe = getLangOpts().ThreadsafeStatics &&
1687 (info.getLocal() || nonTemplateInline) &&
1688 info.getTls() == cir::TLSKind::None;
1689
1690 // If we have a global variable with internal linkage and thread-safe statics
1691 // are disabled, we can just let the guard variable be of type i8.
1692 bool useInt8GuardVariable = !threadsafe && globalOp.hasInternalLinkage();
1693
1694 // Create the guard variable if we don't already have it.
1695 cir::GlobalOp guard = getOrCreateStaticLocalDeclGuardAddress(
1696 builder, globalOp, guardName: globalOp.getDynamicInitGuard()->getName().getValue(),
1697 isLocalVarDecl: info.getLocal(), useInt8GuardVariable);
1698 if (!guard) {
1699 return;
1700 }
1701
1702 mlir::Value guardPtr =
1703 builder.createGetGlobal(global: guard, threadLocal: info.getTls() != cir::TLSKind::None);
1704
1705 // Test whether the variable has completed initialization.
1706 //
1707 // Itanium C++ ABI 3.3.2:
1708 // The following is pseudo-code showing how these functions can be used:
1709 // if (obj_guard.first_byte == 0) {
1710 // if ( __cxa_guard_acquire (&obj_guard) ) {
1711 // try {
1712 // ... initialize the object ...;
1713 // } catch (...) {
1714 // __cxa_guard_abort (&obj_guard);
1715 // throw;
1716 // }
1717 // ... queue object destructor with __cxa_atexit() ...;
1718 // __cxa_guard_release (&obj_guard);
1719 // }
1720 // }
1721 //
1722 // If threadsafe statics are enabled, but we don't have inline atomics, just
1723 // call __cxa_guard_acquire unconditionally. The "inline" check isn't
1724 // actually inline, and the user might not expect calls to __atomic libcalls.
1725 unsigned maxInlineWidthInBits = getTargetInfo().getMaxAtomicInlineWidth();
1726
1727 if (!threadsafe || maxInlineWidthInBits) {
1728 // Load the first byte of the guard variable.
1729 auto bytePtrTy = cir::PointerType::get(pointee: builder.getSIntNTy(n: 8));
1730 mlir::Value bytePtr = builder.createBitcast(src: guardPtr, newTy: bytePtrTy);
1731 mlir::Value guardLoad = builder.createAlignedLoad(
1732 loc: globalOp.getLoc(), ptr: bytePtr, alignment: *guard.getAlignment());
1733
1734 // Itanium ABI:
1735 // An implementation supporting thread-safety on multiprocessor
1736 // systems must also guarantee that references to the initialized
1737 // object do not occur before the load of the initialization flag.
1738 //
1739 // In LLVM, we do this by marking the load Acquire.
1740 if (threadsafe) {
1741 auto loadOp = mlir::cast<cir::LoadOp>(Val: guardLoad.getDefiningOp());
1742 loadOp.setMemOrder(cir::MemOrder::Acquire);
1743 loadOp.setSyncScope(cir::SyncScopeKind::System);
1744 }
1745
1746 // For ARM, we should only check the first bit, rather than the entire byte:
1747 //
1748 // ARM C++ ABI 3.2.3.1:
1749 // To support the potential use of initialization guard variables
1750 // as semaphores that are the target of ARM SWP and LDREX/STREX
1751 // synchronizing instructions we define a static initialization
1752 // guard variable to be a 4-byte aligned, 4-byte word with the
1753 // following inline access protocol.
1754 // #define INITIALIZED 1
1755 // if ((obj_guard & INITIALIZED) != INITIALIZED) {
1756 // if (__cxa_guard_acquire(&obj_guard))
1757 // ...
1758 // }
1759 //
1760 // and similarly for ARM64:
1761 //
1762 // ARM64 C++ ABI 3.2.2:
1763 // This ABI instead only specifies the value bit 0 of the static guard
1764 // variable; all other bits are platform defined. Bit 0 shall be 0 when
1765 // the variable is not initialized and 1 when it is.
1766 if (useARMGuardVarABI() && !useInt8GuardVariable) {
1767 auto one = builder.getConstantInt(
1768 loc: globalOp.getLoc(), ty: mlir::cast<cir::IntType>(Val: guardLoad.getType()), value: 1);
1769 guardLoad = builder.createAnd(loc: globalOp.getLoc(), lhs: guardLoad, rhs: one);
1770 }
1771
1772 // Check if the first byte of the guard variable is zero.
1773 auto zero = builder.getConstantInt(
1774 loc: globalOp.getLoc(), ty: mlir::cast<cir::IntType>(Val: guardLoad.getType()), value: 0);
1775 auto needsInit = builder.createCompare(loc: globalOp.getLoc(),
1776 kind: cir::CmpOpKind::eq, lhs: guardLoad, rhs: zero);
1777
1778 // Build the guarded initialization inside an if block.
1779 cir::IfOp::create(
1780 builder, location: globalOp.getLoc(), cond: needsInit,
1781 /*withElseRegion=*/false, thenBuilder: [&](mlir::OpBuilder &, mlir::Location) {
1782 emitCXXGuardedInitIf(
1783 builder, globalOp, ctorRegion, dtorRegion, isLocalVarDecl: info.getLocal(),
1784 guardPtr, guardPtrTy: builder.getPointerTo(ty: guard.getSymType()), threadsafe);
1785 });
1786 } else {
1787 // Threadsafe statics without inline atomics - call __cxa_guard_acquire
1788 // unconditionally without the initial guard byte check.
1789 globalOp->emitError(message: "NYI: guarded init without inline atomics support");
1790 return;
1791 }
1792}
1793
1794void LoweringPreparePass::lowerLocalInitOp(cir::LocalInitOp initOp) {
1795
1796 // If we don't actually need to initialize anything anymore, we're done here.
1797 if (initOp.getCtorRegion().empty() && initOp.getDtorRegion().empty()) {
1798 initOp.erase();
1799 return;
1800 }
1801
1802 cir::GlobalOp globalOp = initOp.getReferencedGlobal(tables&: symbolTables);
1803 assert(globalOp && "No global-op found");
1804
1805 handleStaticLocal(globalOp, localInitOp: initOp);
1806
1807 // Remove the init local op, now that we've done everything we need with it.
1808 initOp.erase();
1809}
1810static bool isThreadWrapperReplaceable(const clang::TargetInfo &targetInfo) {
1811 // Note: Classic codegen needs to check that the VarDecl.getTLSKind() ==
1812 // TLS_Dynamic, but we don't attempt to emit the thread wrapper unless that is
1813 // already the case. So the only thing that matters here is whether it is
1814 // darwin.
1815 return targetInfo.getTriple().isOSDarwin();
1816}
1817
1818static cir::GlobalLinkageKind
1819getThreadLocalWrapperLinkage(GlobalOp op, const clang::TargetInfo &targetInfo) {
1820 if (isLocalLinkage(linkage: op.getLinkage()))
1821 return op.getLinkage();
1822
1823 if (isThreadWrapperReplaceable(targetInfo))
1824 if (!isLinkOnceLinkage(linkage: op.getLinkage()) &&
1825 !isWeakODRLinkage(linkage: op.getLinkage()))
1826 return op.getLinkage();
1827
1828 // If this isn't a TU in which this variable is defined, the thread wrapper is
1829 // discardable.
1830 if (op.isDeclaration())
1831 return cir::GlobalLinkageKind::LinkOnceODRLinkage;
1832 return cir::GlobalLinkageKind::WeakODRLinkage;
1833}
1834
1835cir::FuncOp
1836LoweringPreparePass::getOrCreateThreadLocalWrapper(CIRBaseBuilderTy &builder,
1837 GlobalOp op) {
1838 mlir::OpBuilder::InsertionGuard insertGuard(builder);
1839 builder.setInsertionPointToStart(&mlirModule.getBodyRegion().front());
1840
1841 mlir::StringAttr wrapperName = op.getTlsRefs()->getWrapperName();
1842
1843 auto existingWrapperIter = threadLocalWrappers.find(Key: wrapperName.getValue());
1844 if (existingWrapperIter != threadLocalWrappers.end())
1845 return existingWrapperIter->second;
1846
1847 // type is ptr-to-global-type(void);
1848 auto funcType = cir::FuncType::get(inputs: {}, returnType: builder.getPointerTo(ty: op.getSymType()));
1849 cir::FuncOp func =
1850 cir::FuncOp::create(builder, location: op.getLoc(), sym_name: wrapperName, type: funcType);
1851
1852 cir::GlobalLinkageKind linkageKind =
1853 getThreadLocalWrapperLinkage(op, targetInfo: getTargetInfo());
1854 func.setLinkageAttr(
1855 cir::GlobalLinkageKindAttr::get(context: &getContext(), value: linkageKind));
1856
1857 if (getTargetInfo().getTriple().supportsCOMDAT() && func.isWeakForLinker())
1858 func.setSelfComdat();
1859
1860 mlir::SymbolTable::setSymbolVisibility(
1861 symbol: func, vis: mlir::SymbolTable::Visibility::Private);
1862
1863 if (!isLocalLinkage(linkage: linkageKind)) {
1864 if (!isThreadWrapperReplaceable(targetInfo: getTargetInfo()) ||
1865 isLinkOnceLinkage(linkage: linkageKind) || isWeakODRLinkage(linkage: linkageKind) ||
1866 op.getGlobalVisibility() == cir::VisibilityKind::Hidden)
1867 func.setGlobalVisibility(cir::VisibilityKind::Hidden);
1868 }
1869 if (isThreadWrapperReplaceable(targetInfo: getTargetInfo()))
1870 op->emitError(message: "Unhandled thread wrapper attributes for CC and Nounwind");
1871
1872 threadLocalWrappers.insert(KV: {wrapperName.getValue(), func});
1873 return func;
1874}
1875
1876void LoweringPreparePass::defineGlobalThreadLocalWrapper(cir::GlobalOp op,
1877 cir::FuncOp initAlias,
1878 bool isVarDefinition) {
1879 CIRBaseBuilderTy builder(getContext());
1880 cir::FuncOp wrapper = getOrCreateThreadLocalWrapper(builder, op);
1881 mlir::Block *entryBB = wrapper.addEntryBlock();
1882 builder.setInsertionPointToStart(entryBB);
1883 // If we are a situation where we have/need one, emit a call to the init
1884 // function.
1885 if (initAlias) {
1886 mlir::Location aliasLoc = initAlias.getLoc();
1887 if (!isVarDefinition) {
1888 // If this isn't a definition, we have to check that the alias exists.
1889 mlir::Value funcLoad = cir::GetGlobalOp::create(
1890 builder, location: aliasLoc, addr: cir::PointerType::get(pointee: initAlias.getFunctionType()),
1891 name: initAlias.getSymName());
1892 mlir::Value nullCheck =
1893 builder.getNullValue(ty: funcLoad.getType(), loc: aliasLoc);
1894 mlir::Value cmp = cir::CmpOp::create(
1895 builder, location: aliasLoc, kind: cir::CmpOpKind::ne, lhs: funcLoad, rhs: nullCheck);
1896 cir::IfOp::create(builder, location: aliasLoc, cond: cmp, /*withElseRegion=*/false,
1897 thenBuilder: [&](mlir::OpBuilder &, mlir::Location loc) {
1898 builder.createCallOp(loc: aliasLoc, callee: initAlias, operands: {},
1899 /*attrs=*/{},
1900 /*argAttrs=*/{}, /*resAttrs=*/{},
1901 callingConv: initAlias.getCallingConv());
1902 cir::YieldOp::create(builder, location: aliasLoc);
1903 });
1904 } else {
1905 // If this IS a definition, we know the alias exists, so we can just emit
1906 // a call to it.
1907 builder.createCallOp(loc: aliasLoc, callee: initAlias, operands: {}, /*attrs=*/{},
1908 /*argAttrs=*/{}, /*resAttrs=*/{},
1909 callingConv: initAlias.getCallingConv());
1910 }
1911 }
1912 cir::GetGlobalOp get = builder.createGetGlobal(global: op, /*tls=*/threadLocal: true);
1913 cir::ReturnOp::create(builder, location: op.getLoc(), input: {get});
1914}
1915
1916cir::FuncOp
1917LoweringPreparePass::defineGlobalThreadLocalInitAlias(cir::GlobalOp op,
1918 cir::FuncOp aliasee) {
1919 CIRBaseBuilderTy builder(getContext());
1920 mlir::OpBuilder::InsertionGuard insertGuard(builder);
1921 builder.setInsertionPointToStart(&mlirModule.getBodyRegion().front());
1922 mlir::StringAttr aliasName = op.getTlsRefs()->getInitName();
1923 auto existingAliasIter = threadLocalInitAliases.find(Key: aliasName.getValue());
1924
1925 if (existingAliasIter != threadLocalInitAliases.end())
1926 return existingAliasIter->second;
1927
1928 auto funcType = builder.getVoidFnTy();
1929 cir::FuncOp alias =
1930 cir::FuncOp::create(builder, location: op.getLoc(), sym_name: aliasName, type: funcType);
1931 alias.setLinkage(op.getLinkage());
1932
1933 if (aliasee) {
1934 alias.setAliasee(aliasee.getSymName());
1935 } else {
1936 // If we don't have anything to alias (because this isn't a variable
1937 // definition!), we set this as just a function definition with no alias,
1938 // and extern-weak.
1939 alias.setLinkage(cir::GlobalLinkageKind::ExternalWeakLinkage);
1940 mlir::SymbolTable::setSymbolVisibility(
1941 symbol: alias, vis: mlir::SymbolTable::Visibility::Private);
1942 }
1943
1944 threadLocalInitAliases.insert(KV: {aliasName.getValue(), alias});
1945 return alias;
1946}
1947
1948void LoweringPreparePass::lowerGlobalOp(GlobalOp op) {
1949 // Static locals are handled separately via guard variables.
1950 if (op.getDynamicInitGuard() &&
1951 (!op.getDynamicInitInfo() || op.getDynamicInitInfo()->getLocal()))
1952 return;
1953
1954 mlir::Region &ctorRegion = op.getCtorRegion();
1955 mlir::Region &dtorRegion = op.getDtorRegion();
1956 cir::FuncOp initAlias;
1957
1958 if (!ctorRegion.empty() || !dtorRegion.empty()) {
1959 // Build a variable initialization function and move the initialzation code
1960 // in the ctor region over.
1961 cir::FuncOp f = buildCXXGlobalVarDeclInitFunc(op);
1962
1963 // Clear the ctor and dtor region
1964 ctorRegion.getBlocks().clear();
1965 dtorRegion.getBlocks().clear();
1966
1967 assert(!cir::MissingFeatures::astVarDeclInterface());
1968 if (op.getTlsModel()) {
1969 // There are two types of global TLS variables: 'ordered' and 'unordered'.
1970 // 'ordered' are the common case. A call to any of them causes all of the
1971 // initializers for all other 'ordered' ones to be called, via a
1972 // `__tls_init` function. So the 'init alias' that gets called in the
1973 // wrapper for these goes directly to `__tls_init`.
1974
1975 // 'Unordered' values are the case for variable templates. In this case,
1976 // their init alias goes directly to their init function. The FE generates
1977 // a guard variable for them (since they cannot use the global guard), so
1978 // we differentiate them that way.
1979
1980 if (op.getTlsRefs()->getGuardName()) {
1981 // Unordered: the alias is the function we just generated.
1982 initAlias = defineGlobalThreadLocalInitAlias(op, aliasee: f);
1983 } else {
1984 // Ordered: Get the __tls_init, and make the alias to that.
1985 initAlias = defineGlobalThreadLocalInitAlias(op, aliasee: getTlsInitFn());
1986 // Ordered inits also need to get called from the __tls_init function,
1987 // so we add the init function to the list, so that we can add them to
1988 // it later.
1989 globalThreadLocalInitializers.push_back(Elt: f);
1990 }
1991 } else if (std::optional<uint32_t> priority = op.getInitPriority()) {
1992 prioritizedDynamicInitializers[*priority].push_back(Elt: f);
1993 } else if (op.getDynamicInitInfo() &&
1994 (op.getDynamicInitInfo()->getIsInline() ||
1995 op.getDynamicInitInfo()->isTemplateInstantiation()) &&
1996 getTargetInfo().getTriple().supportsCOMDAT() &&
1997 !op.hasInternalLinkage()) {
1998 // Vague-linkage globals (inline variables and template instantiations)
1999 // get their own `llvm.global_ctors` entry keyed to the variable via
2000 // COMDAT associated data, rather than being folded into
2001 // `_GLOBAL__sub_I_*`. This ensures at most one initializer per DSO runs,
2002 // and lets the linker drop the initializer call along with a discarded
2003 // duplicate definition of the variable. Other guarded globals, such as a
2004 // plain weak variable, stay ordered with the rest of the translation
2005 // unit's initializers. Matches classic CodeGen's
2006 // EmitCXXGlobalVarDeclInitFunc.
2007 assert(!cir::MissingFeatures::msabi());
2008 globalCtorList.emplace_back(Args: f.getSymName(),
2009 Args: cir::GlobalCtorAttr::getDefaultPriority(),
2010 Args: op.getSymName());
2011 // When COMDAT is used on ELF or in the MS C++ ABI, the key must be in
2012 // `llvm.used` to prevent the linker from garbage-collecting it (and,
2013 // with it, the global ctor entry keyed to it). Matches classic
2014 // CodeGen's EmitCXXGlobalVarDeclInitFunc.
2015 if (getTargetInfo().getTriple().isOSBinFormatELF() ||
2016 getTargetInfo().getCXXABI().isMicrosoft())
2017 addUsedGlobal(global: op);
2018
2019 // Since the global ctor entry is keyed to the variable, the init function
2020 // can be discarded along with it, so put it in the variable's COMDAT.
2021 std::optional<llvm::StringRef> comdat = op.getComdat();
2022 if (comdat && (getTargetInfo().getTriple().isOSBinFormatELF() ||
2023 getTargetInfo().getTriple().isOSBinFormatWasm()))
2024 f.setComdat(comdat->empty() ? op.getSymName() : *comdat);
2025 } else {
2026 dynamicInitializers.push_back(Elt: f);
2027 }
2028 } else if (op.getTlsModel() && op.getTlsRefs() && op.isDeclaration()) {
2029 // If this is a declaration and has no init function, we probably DO have to
2030 // create an alias that needs checking, so create it as extern-weak.
2031 initAlias = defineGlobalThreadLocalInitAlias(op, aliasee: {});
2032 }
2033
2034 // We need a wrapper for TLS globals that MIGHT have a non-constant
2035 // initialization. The FE will have generated the TlsRefs for any with
2036 // known dynamic init, or unknown (extern) init.
2037 if (op.getTlsModel() && op.getTlsRefs())
2038 defineGlobalThreadLocalWrapper(op, initAlias, isVarDefinition: !op.isDeclaration());
2039
2040 assert(!cir::MissingFeatures::opGlobalAnnotations());
2041}
2042
2043void LoweringPreparePass::lowerGetGlobalOp(GetGlobalOp op) {
2044 if (!op.getTls())
2045 return;
2046 auto globalOp = mlir::cast<cir::GlobalOp>(
2047 Val: symbolTables.lookupNearestSymbolFrom(from: op, symbol: op.getNameAttr()));
2048
2049 // Only global/namespace scope thread local variables need to have their
2050 // get-global operations rewritten to be calls to a wrapper function. If
2051 // we're not in a dynamic TLS (or one without the TLS markers), we can leave
2052 // this one as a get-global and return early.
2053 if (!globalOp.getTlsModel() || !globalOp.getTlsRefs())
2054 return;
2055
2056 // If this is a global TLS, we need to replace the call to 'get_global' with a
2057 // call to the wrapper function. Classic codegen figures out some cases where
2058 // we can omit this, but for now we're going to always put it in, as it is
2059 // effectively a no-op.
2060
2061 // The first 'GetGlobalOp' at the beginning of a ctor/dtor region on one of
2062 // these is for the purpose of creating/destroying. We want to skip replacing
2063 // THAT one, but leave all other get-global-ops in place, else
2064 // self-referential ops won't work right.
2065
2066 // Note that ctors/dtors are removed during this pass. We get away with these
2067 // checks because the only time that these situations can actually be true
2068 // (that is, the ctor/dtor region exist) is if we're in the process of
2069 // converting the ctor/dtor for this. If we're NOT doing that, the ctor/dtor
2070 // will have already disappeared.
2071 mlir::Operation *parentOp = op->getParentOp();
2072 if (parentOp == globalOp) {
2073 mlir::Region *ctorRegion = &globalOp.getCtorRegion();
2074 mlir::Region *dtorRegion = &globalOp.getDtorRegion();
2075
2076 if (!ctorRegion->empty() && &*ctorRegion->op_begin() == op.getOperation())
2077 return;
2078 if (!dtorRegion->empty() && &*dtorRegion->op_begin() == op.getOperation())
2079 return;
2080 }
2081
2082 CIRBaseBuilderTy builder(getContext());
2083 cir::FuncOp wrapperFunc = getOrCreateThreadLocalWrapper(builder, op: globalOp);
2084
2085 builder.setInsertionPoint(op);
2086 cir::CallOp call = builder.createCallOp(
2087 loc: wrapperFunc.getLoc(),
2088 callee: mlir::FlatSymbolRefAttr::get(value: wrapperFunc.getSymNameAttr()),
2089 returnType: wrapperFunc.getFunctionType().getReturnType(), operands: {}, /*attrs=*/{},
2090 /*argAttrs=*/{}, /*resAttrs=*/{}, callingConv: wrapperFunc.getCallingConv());
2091 op->replaceAllUsesWith(values&: call);
2092 op.erase();
2093}
2094
2095void LoweringPreparePass::lowerThreeWayCmpOp(CmpThreeWayOp op) {
2096 CIRBaseBuilderTy builder(getContext());
2097 builder.setInsertionPointAfter(op);
2098
2099 mlir::Location loc = op->getLoc();
2100 cir::CmpThreeWayInfoAttr cmpInfo = op.getInfo();
2101
2102 mlir::Value ltRes =
2103 builder.getConstantInt(loc, ty: op.getType(), value: cmpInfo.getLt());
2104 mlir::Value eqRes =
2105 builder.getConstantInt(loc, ty: op.getType(), value: cmpInfo.getEq());
2106 mlir::Value gtRes =
2107 builder.getConstantInt(loc, ty: op.getType(), value: cmpInfo.getGt());
2108
2109 mlir::Value transformedResult;
2110 if (cmpInfo.getOrdering() != CmpOrdering::Partial) {
2111 // Total ordering
2112 mlir::Value lt =
2113 builder.createCompare(loc, kind: CmpOpKind::lt, lhs: op.getLhs(), rhs: op.getRhs());
2114 mlir::Value selectOnLt = builder.createSelect(loc, condition: lt, trueValue: ltRes, falseValue: gtRes);
2115 mlir::Value eq =
2116 builder.createCompare(loc, kind: CmpOpKind::eq, lhs: op.getLhs(), rhs: op.getRhs());
2117 transformedResult = builder.createSelect(loc, condition: eq, trueValue: eqRes, falseValue: selectOnLt);
2118 } else {
2119 // Partial ordering
2120 cir::ConstantOp unorderedRes = builder.getConstantInt(
2121 loc, ty: op.getType(), value: cmpInfo.getUnordered().value());
2122
2123 mlir::Value eq =
2124 builder.createCompare(loc, kind: CmpOpKind::eq, lhs: op.getLhs(), rhs: op.getRhs());
2125 mlir::Value selectOnEq = builder.createSelect(loc, condition: eq, trueValue: eqRes, falseValue: unorderedRes);
2126 mlir::Value gt =
2127 builder.createCompare(loc, kind: CmpOpKind::gt, lhs: op.getLhs(), rhs: op.getRhs());
2128 mlir::Value selectOnGt = builder.createSelect(loc, condition: gt, trueValue: gtRes, falseValue: selectOnEq);
2129 mlir::Value lt =
2130 builder.createCompare(loc, kind: CmpOpKind::lt, lhs: op.getLhs(), rhs: op.getRhs());
2131 transformedResult = builder.createSelect(loc, condition: lt, trueValue: ltRes, falseValue: selectOnGt);
2132 }
2133
2134 op.replaceAllUsesWith(newValue: transformedResult);
2135 op.erase();
2136}
2137
2138template <typename AttributeTy>
2139static llvm::SmallVector<mlir::Attribute>
2140prepareCtorDtorAttrList(mlir::MLIRContext *context,
2141 llvm::ArrayRef<cir::GlobalCtorDtorEntry> list) {
2142 llvm::SmallVector<mlir::Attribute> attrs;
2143 for (const auto &entry : list) {
2144 mlir::FlatSymbolRefAttr associated;
2145 if (!entry.associated.empty())
2146 associated = mlir::FlatSymbolRefAttr::get(ctx: context, value: entry.associated);
2147 attrs.push_back(
2148 Elt: AttributeTy::get(context, entry.name, entry.priority, associated));
2149 }
2150 return attrs;
2151}
2152
2153void LoweringPreparePass::buildGlobalCtorDtorList() {
2154 if (!globalCtorList.empty()) {
2155 llvm::SmallVector<mlir::Attribute> globalCtors =
2156 prepareCtorDtorAttrList<cir::GlobalCtorAttr>(context: &getContext(),
2157 list: globalCtorList);
2158
2159 mlirModule->setAttr(name: cir::CIRDialect::getGlobalCtorsAttrName(),
2160 value: mlir::ArrayAttr::get(context: &getContext(), value: globalCtors));
2161 }
2162
2163 if (!globalDtorList.empty()) {
2164 llvm::SmallVector<mlir::Attribute> globalDtors =
2165 prepareCtorDtorAttrList<cir::GlobalDtorAttr>(context: &getContext(),
2166 list: globalDtorList);
2167 mlirModule->setAttr(name: cir::CIRDialect::getGlobalDtorsAttrName(),
2168 value: mlir::ArrayAttr::get(context: &getContext(), value: globalDtors));
2169 }
2170}
2171
2172cir::GlobalOp
2173LoweringPreparePass::createGlobalThreadLocalGuard(CIRBaseBuilderTy &builder,
2174 mlir::Location loc) {
2175 mlir::OpBuilder::InsertionGuard guard(builder);
2176 builder.setInsertionPointToStart(mlirModule.getBody());
2177
2178 // The TLS Guard is always an Int8Ty.
2179 cir::IntType guardTy = builder.getSIntNTy(n: 8);
2180 auto g = cir::GlobalOp::create(builder, location: loc, sym_name: "__tls_guard", sym_type: guardTy);
2181 g.setLinkageAttr(cir::GlobalLinkageKindAttr::get(
2182 context: builder.getContext(), value: cir::GlobalLinkageKind::InternalLinkage));
2183 g.setAlignment(clang::CharUnits::One().getAsAlign().value());
2184
2185 if (auto defTlsModel = mlirModule->getAttrOfType<TLSModelAttr>(
2186 name: cir::CIRDialect::getDefaultTlsModelAttrName())) {
2187 g.setTlsModel(defTlsModel.getValue());
2188 } else {
2189 // Default value, unless overridden in the IR/by the frontend.
2190 g.setTlsModel(TLSModel::GeneralDynamic);
2191 }
2192
2193 g.setInitialValueAttr(cir::IntAttr::get(type: guardTy, value: 0));
2194 return g;
2195}
2196
2197cir::IfOp LoweringPreparePass::buildInitGuardCheck(CIRBaseBuilderTy &builder,
2198 mlir::Location loc,
2199 cir::GlobalOp guard,
2200 bool threadLocal) {
2201 cir::GetGlobalOp getGuard = builder.createGetGlobal(global: guard, threadLocal);
2202 mlir::Value getGuardValue = getGuard;
2203
2204 // Classic codegen always just loads the first byte of the guard instead of
2205 // the whole thing. __tls_guard is already only 8 bits, but for the case of
2206 // unordered TLS, it gets created as 64 bits.
2207 if (guard.getSymType() != builder.getSIntNTy(n: 8))
2208 getGuardValue = builder.createBitcast(
2209 src: getGuard, newTy: cir::PointerType::get(pointee: builder.getSIntNTy(n: 8)));
2210
2211 mlir::Value guardLoad =
2212 builder.createAlignedLoad(loc, ptr: getGuardValue, alignment: *guard.getAlignment());
2213 auto zero = builder.getConstantInt(loc, ty: builder.getSIntNTy(n: 8), value: 0);
2214 cir::CmpOp compare =
2215 builder.createCompare(loc, kind: cir::CmpOpKind::eq, lhs: guardLoad, rhs: zero);
2216 return cir::IfOp::create(
2217 builder, location: loc, cond: compare,
2218 /*withElseRegion=*/false, thenBuilder: [&](mlir::OpBuilder &, mlir::Location loc) {
2219 // Classic codegen still does this store as a i8, but it doesn't seem
2220 // reasonable to do an i8 store into a 64 bit value?
2221 builder.createStore(
2222 loc, val: builder.getConstantInt(loc, ty: guard.getSymType(), value: 1), dst: getGuard);
2223 });
2224}
2225
2226void LoweringPreparePass::buildCXXGlobalTlsFunc() {
2227 if (globalThreadLocalInitializers.empty())
2228 return;
2229
2230 // The global-ordered-init function for TLS variables just calls each of the
2231 // init-functions in order after doing a guard.
2232
2233 cir::FuncOp tlsInit = getTlsInitFn();
2234 mlir::Location loc = tlsInit.getLoc();
2235 CIRBaseBuilderTy builder(getContext());
2236 mlir::Block *entryBB = tlsInit.addEntryBlock();
2237 builder.setInsertionPointToStart(entryBB);
2238
2239 cir::IfOp ifOperation = buildInitGuardCheck(
2240 builder, loc, guard: createGlobalThreadLocalGuard(builder, loc),
2241 /*threadLocal=*/true);
2242
2243 // Emit the body of the guarded spot.
2244 builder.setInsertionPointToEnd(&ifOperation.getThenRegion().front());
2245 for (cir::FuncOp initFunc : globalThreadLocalInitializers)
2246 builder.createCallOp(loc, callee: initFunc, operands: {}, /*attrs=*/{}, /*argAttrs=*/{},
2247 /*resAttrs=*/{}, callingConv: initFunc.getCallingConv());
2248 cir::YieldOp::create(builder, location: loc);
2249
2250 builder.setInsertionPointAfter(ifOperation);
2251 cir::ReturnOp::create(builder, location: loc);
2252}
2253
2254/// Compute the zero-padded priority suffix used to name priority-specific
2255/// global init functions, so that the function names also sort in priority
2256/// order (e.g. 200 -> "000200").
2257static std::string getPrioritySuffix(unsigned priority) {
2258 assert(priority <= 65535 && "Priority should always be <= 65535.");
2259 std::string prioritySuffix = llvm::utostr(X: priority);
2260 assert(prioritySuffix.size() < 6);
2261 prioritySuffix = std::string(6 - prioritySuffix.size(), '0') + prioritySuffix;
2262 return prioritySuffix;
2263}
2264
2265cir::FuncOp LoweringPreparePass::buildGlobalInitCallerFunc(
2266 llvm::StringRef fnName, cir::GlobalLinkageKind linkage,
2267 llvm::ArrayRef<cir::FuncOp> initializers, uint32_t priority,
2268 cir::GlobalOp guard) {
2269 CIRBaseBuilderTy builder(getContext());
2270 builder.setInsertionPointToEnd(&mlirModule.getBodyRegion().back());
2271 auto fnType = cir::FuncType::get(inputs: {}, returnType: builder.getVoidTy());
2272 cir::FuncOp fn =
2273 buildRuntimeFunction(builder, name: fnName, loc: mlirModule.getLoc(), type: fnType,
2274 linkage, /*hasDefaultFuncAttrs=*/true);
2275 builder.setInsertionPointToStart(fn.addEntryBlock());
2276 mlir::Location loc = fn.getLoc();
2277 auto emitCalls = [&] {
2278 for (cir::FuncOp init : initializers)
2279 builder.createCallOp(loc: init.getLoc(), callee: init, operands: {}, /*attrs=*/{},
2280 /*argAttrs=*/{}, /*resAttrs=*/{},
2281 callingConv: init.getCallingConv());
2282 };
2283 if (guard) {
2284 cir::IfOp guardIf =
2285 buildInitGuardCheck(builder, loc, guard, /*threadLocal=*/false);
2286 builder.setInsertionPointToEnd(&guardIf.getThenRegion().front());
2287 emitCalls();
2288 cir::YieldOp::create(builder, location: loc);
2289 builder.setInsertionPointAfter(guardIf);
2290 } else {
2291 emitCalls();
2292 }
2293 cir::ReturnOp::create(builder, location: loc);
2294 globalCtorList.emplace_back(Args&: fnName, Args&: priority);
2295 return fn;
2296}
2297
2298void LoweringPreparePass::buildCXXGlobalPriorityInitFuncs(
2299 llvm::SmallVectorImpl<cir::FuncOp> &importedInits) {
2300 // std::map keeps priorities in ascending order, so each group is already
2301 // ready to emit into its own function, named after its priority so that
2302 // the functions are naturally ordered relative to one another.
2303 for (const auto &[priority, initializers] : prioritizedDynamicInitializers) {
2304 SmallString<256> fnName;
2305 fnName += "_GLOBAL__I_";
2306 fnName += getPrioritySuffix(priority);
2307
2308 llvm::SmallVector<cir::FuncOp> calls(importedInits.begin(),
2309 importedInits.end());
2310 calls.append(in_start: initializers.begin(), in_end: initializers.end());
2311 importedInits.clear();
2312 buildGlobalInitCallerFunc(fnName, linkage: cir::GlobalLinkageKind::InternalLinkage,
2313 initializers: calls, priority);
2314 }
2315}
2316
2317void LoweringPreparePass::buildCXXGlobalInitFunc() {
2318 CIRBaseBuilderTy builder(getContext());
2319 builder.setInsertionPointToEnd(&mlirModule.getBodyRegion().back());
2320 mlir::Location loc = mlirModule.getLoc();
2321
2322 // The initializers of the imported C++20 named modules, recorded by CIRGen
2323 // as the mangled names of functions this translation unit declares, at the
2324 // location of the import, and calls before its own initializers.
2325 llvm::SmallVector<cir::FuncOp> importedInits;
2326 if (auto inits = mlirModule->getAttrOfType<mlir::ArrayAttr>(
2327 name: cir::CIRDialect::getCXXModuleImportedInitsAttrName()))
2328 for (auto init : inits.getAsRange<cir::CXXModuleInitAttr>())
2329 importedInits.push_back(
2330 Elt: buildRuntimeFunction(builder, name: init.getName().getValue(),
2331 loc: init.getLoc(), type: builder.getVoidFnTy()));
2332
2333 // The C++20 named-module init function name is precomputed by CIRGen and
2334 // stored as a module-level attribute. Its presence is what marks this
2335 // module as a named-module interface unit, so the name and the external
2336 // linkage that goes with it both come from the attribute and this pass needs
2337 // no live ASTContext. Modules built directly from textual CIR can opt in to
2338 // the module-init form by setting the same attribute.
2339 if (auto moduleInitFnName = mlirModule->getAttrOfType<mlir::StringAttr>(
2340 name: cir::CIRDialect::getCXXModuleInitFnNameAttrName())) {
2341 // As in classic CodeGen's EmitCXXModuleInitFunc, an interface unit gets
2342 // one initializer function, even with nothing to run, since importing
2343 // translation units may call it. It runs the imported modules'
2344 // initializers, then the unit's own in priority order, which get no
2345 // `_GLOBAL__I_<priority>` function of their own. The global ctor entry
2346 // and every importing translation unit call it, so a guard byte makes
2347 // the initializers run once; there is none when there is nothing to run.
2348 llvm::SmallVector<cir::FuncOp> initializers(importedInits.begin(),
2349 importedInits.end());
2350 for (const auto &[priority, inits] : prioritizedDynamicInitializers)
2351 initializers.append(in_start: inits.begin(), in_end: inits.end());
2352 initializers.append(in_start: dynamicInitializers.begin(), in_end: dynamicInitializers.end());
2353
2354 llvm::StringRef fnName = moduleInitFnName.getValue();
2355 cir::GlobalOp guard;
2356 if (!initializers.empty()) {
2357 cir::IntType guardTy = builder.getSIntNTy(n: 8);
2358 guard =
2359 createGuardGlobalOp(builder, loc, name: (fnName + "__in_chrg").str(),
2360 guardTy, linkage: cir::GlobalLinkageKind::InternalLinkage);
2361 guard.setAlignment(clang::CharUnits::One().getAsAlign().value());
2362 guard.setInitialValueAttr(cir::IntAttr::get(type: guardTy, value: 0));
2363 }
2364 buildGlobalInitCallerFunc(fnName, linkage: cir::GlobalLinkageKind::ExternalLinkage,
2365 initializers,
2366 priority: cir::GlobalCtorAttr::getDefaultPriority(), guard);
2367 return;
2368 }
2369
2370 // Any other translation unit, as in classic CodeGen's
2371 // EmitCXXGlobalInitFunc: one function per priority, the imported modules'
2372 // initializers in front of the first, then the default-priority function
2373 // for the rest, only when there is something left to call.
2374 buildCXXGlobalPriorityInitFuncs(importedInits);
2375 if (importedInits.empty() && dynamicInitializers.empty())
2376 return;
2377
2378 llvm::SmallVector<cir::FuncOp> initializers(importedInits.begin(),
2379 importedInits.end());
2380 initializers.append(in_start: dynamicInitializers.begin(), in_end: dynamicInitializers.end());
2381
2382 // Include the filename in the symbol name. Including "sub_" matches gcc
2383 // and makes sure these symbols appear lexicographically behind the symbols
2384 // with priority (TBD). Module implementation units behave the same
2385 // way as a non-modular TU with imports.
2386 SmallString<256> fnName;
2387 fnName += "_GLOBAL__sub_I_";
2388 fnName += getTransformedFileName(mlirModule);
2389 buildGlobalInitCallerFunc(fnName, linkage: cir::GlobalLinkageKind::InternalLinkage,
2390 initializers,
2391 priority: cir::GlobalCtorAttr::getDefaultPriority());
2392}
2393
2394/// Lower a cir.array.ctor or cir.array.dtor into a do-while loop that
2395/// iterates over every element. For cir.array.ctor ops whose partial_dtor
2396/// region is non-empty, the ctor loop is wrapped in a cir.cleanup.scope whose
2397/// EH cleanup performs a reverse destruction loop using the partial dtor body.
2398static void lowerArrayDtorCtorIntoLoop(cir::CIRBaseBuilderTy &builder,
2399 const clang::TargetInfo &targetInfo,
2400 mlir::Operation *op, mlir::Type eltTy,
2401 mlir::Value addr,
2402 mlir::Value numElements,
2403 uint64_t arrayLen, bool isCtor) {
2404 mlir::Location loc = op->getLoc();
2405 bool isDynamic = numElements != nullptr;
2406
2407 // TODO: instead of getting the size from the target, create alias for
2408 // PtrDiffTy and unify with CIRGen stuff.
2409 const unsigned sizeTypeSize =
2410 targetInfo.getTypeWidth(T: targetInfo.getSignedSizeType());
2411
2412 // Both constructors and destructors use end = begin + numElements.
2413 // Constructors iterate forward [begin, end). Destructors iterate backward
2414 // from end, decrementing before calling the destructor on each element.
2415 mlir::Value begin, end;
2416 if (isDynamic) {
2417 begin = addr;
2418 end = cir::PtrStrideOp::create(builder, location: loc, result: eltTy, base: begin, stride: numElements);
2419 } else {
2420 mlir::Value endOffsetVal =
2421 builder.getUnsignedInt(loc, val: arrayLen, numBits: sizeTypeSize);
2422 begin = cir::CastOp::create(builder, location: loc, result: eltTy,
2423 kind: cir::CastKind::array_to_ptrdecay, src: addr);
2424 end = cir::PtrStrideOp::create(builder, location: loc, result: eltTy, base: begin, stride: endOffsetVal);
2425 }
2426
2427 mlir::Value start = isCtor ? begin : end;
2428 mlir::Value stop = isCtor ? end : begin;
2429
2430 // For dynamic destructors, guard against zero elements.
2431 // This places the destructor loop emitted below inside the if block.
2432 cir::IfOp ifOp;
2433 if (isDynamic) {
2434 mlir::Value guardCond;
2435 if (isCtor) {
2436 mlir::Value zero = builder.getUnsignedInt(loc, val: 0, numBits: sizeTypeSize);
2437 guardCond = cir::CmpOp::create(builder, location: loc, kind: cir::CmpOpKind::ne,
2438 lhs: numElements, rhs: zero);
2439 } else {
2440 // We could check for numElements != 0 in this case too, but this matches
2441 // what classic codegen does.
2442 guardCond =
2443 cir::CmpOp::create(builder, location: loc, kind: cir::CmpOpKind::ne, lhs: start, rhs: stop);
2444 }
2445 ifOp = cir::IfOp::create(builder, location: loc, cond: guardCond,
2446 /*withElseRegion=*/false,
2447 thenBuilder: [&](mlir::OpBuilder &, mlir::Location) {});
2448 builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
2449 }
2450
2451 cir::CIRDataLayout dataLayout(op->getParentOfType<mlir::ModuleOp>());
2452 cir::PointerType tmpAddrTy = builder.getPointerTo(
2453 ty: eltTy, as: dataLayout.getAllocaAddrSpace(ctx: builder.getContext()));
2454 mlir::Value tmpAddr = builder.createAlloca(
2455 loc, /*addr type*/ addrType: tmpAddrTy, name: "__array_idx",
2456 alignment: builder.getAlignmentAttr(alignment: dataLayout.getABITypeAlign(ty: eltTy).value()));
2457 builder.createStore(loc, val: start, dst: tmpAddr);
2458
2459 mlir::Block *bodyBlock = &op->getRegion(index: 0).front();
2460
2461 // Clone the region body (ctor/dtor call and any setup ops like per-element
2462 // zero-init) into the loop, remapping the block argument to the current
2463 // element pointer.
2464 auto cloneRegionBodyInto = [&](mlir::Block *srcBlock,
2465 mlir::Value replacement) {
2466 mlir::IRMapping map;
2467 map.map(from: srcBlock->getArgument(i: 0), to: replacement);
2468 for (mlir::Operation &regionOp : *srcBlock) {
2469 if (!mlir::isa<cir::YieldOp>(Val: &regionOp))
2470 builder.clone(op&: regionOp, mapper&: map);
2471 }
2472 };
2473
2474 mlir::Block *partialDtorBlock = nullptr;
2475 if (auto arrayCtor = mlir::dyn_cast<cir::ArrayCtor>(Val: op)) {
2476 mlir::Region &partialDtor = arrayCtor.getPartialDtor();
2477 if (!partialDtor.empty())
2478 partialDtorBlock = &partialDtor.front();
2479 } else if (auto arrayDtor = mlir::dyn_cast<cir::ArrayDtor>(Val: op)) {
2480 // When the element destructor may throw, reuse the body block as the
2481 // partial-dtor block so that an exception thrown by an element's dtor
2482 // continues the reverse-destruction loop in the EH cleanup region. The
2483 // body block already stores the next element pointer to `tmpAddr`
2484 // before invoking the dtor, so when an exception unwinds from the
2485 // dtor call `tmpAddr` already points at the element that threw, and
2486 // the cleanup loop picks up from `tmpAddr - 1` and walks back to
2487 // `begin`.
2488 if (arrayDtor.getDtorMayThrow())
2489 partialDtorBlock = bodyBlock;
2490 }
2491
2492 auto emitCtorDtorLoop = [&]() {
2493 builder.createDoWhile(
2494 loc,
2495 /*condBuilder=*/
2496 [&](mlir::OpBuilder &b, mlir::Location loc) {
2497 auto currentElement = cir::LoadOp::create(builder&: b, location: loc, result: eltTy, addr: tmpAddr);
2498 auto cmp = cir::CmpOp::create(builder, location: loc, kind: cir::CmpOpKind::ne,
2499 lhs: currentElement, rhs: stop);
2500 builder.createCondition(condition: cmp);
2501 },
2502 /*bodyBuilder=*/
2503 [&](mlir::OpBuilder &b, mlir::Location loc) {
2504 auto currentElement = cir::LoadOp::create(builder&: b, location: loc, result: eltTy, addr: tmpAddr);
2505 if (isCtor) {
2506 cloneRegionBodyInto(bodyBlock, currentElement);
2507 mlir::Value stride = builder.getUnsignedInt(loc, val: 1, numBits: sizeTypeSize);
2508 auto nextElement = cir::PtrStrideOp::create(builder, location: loc, result: eltTy,
2509 base: currentElement, stride);
2510 builder.createStore(loc, val: nextElement, dst: tmpAddr);
2511 } else {
2512 mlir::Value stride = builder.getSignedInt(loc, val: -1, numBits: sizeTypeSize);
2513 auto prevElement = cir::PtrStrideOp::create(builder, location: loc, result: eltTy,
2514 base: currentElement, stride);
2515 builder.createStore(loc, val: prevElement, dst: tmpAddr);
2516 cloneRegionBodyInto(bodyBlock, prevElement);
2517 }
2518
2519 cir::YieldOp::create(builder&: b, location: loc);
2520 });
2521 };
2522
2523 if (partialDtorBlock) {
2524 cir::CleanupScopeOp::create(
2525 builder, location: loc, cleanupKind: cir::CleanupKind::EH,
2526 /*bodyBuilder=*/
2527 [&](mlir::OpBuilder &b, mlir::Location loc) {
2528 emitCtorDtorLoop();
2529 cir::YieldOp::create(builder&: b, location: loc);
2530 },
2531 /*cleanupBuilder=*/
2532 [&](mlir::OpBuilder &b, mlir::Location loc) {
2533 auto cur = cir::LoadOp::create(builder&: b, location: loc, result: eltTy, addr: tmpAddr);
2534 auto cmp =
2535 cir::CmpOp::create(builder, location: loc, kind: cir::CmpOpKind::ne, lhs: cur, rhs: begin);
2536 cir::IfOp::create(
2537 builder, location: loc, cond: cmp, /*withElseRegion=*/false,
2538 thenBuilder: [&](mlir::OpBuilder &b, mlir::Location loc) {
2539 builder.createDoWhile(
2540 loc,
2541 /*condBuilder=*/
2542 [&](mlir::OpBuilder &b, mlir::Location loc) {
2543 auto el = cir::LoadOp::create(builder&: b, location: loc, result: eltTy, addr: tmpAddr);
2544 auto neq = cir::CmpOp::create(
2545 builder, location: loc, kind: cir::CmpOpKind::ne, lhs: el, rhs: begin);
2546 builder.createCondition(condition: neq);
2547 },
2548 /*bodyBuilder=*/
2549 [&](mlir::OpBuilder &b, mlir::Location loc) {
2550 auto el = cir::LoadOp::create(builder&: b, location: loc, result: eltTy, addr: tmpAddr);
2551 mlir::Value negOne =
2552 builder.getSignedInt(loc, val: -1, numBits: sizeTypeSize);
2553 auto prev = cir::PtrStrideOp::create(builder, location: loc, result: eltTy,
2554 base: el, stride: negOne);
2555 builder.createStore(loc, val: prev, dst: tmpAddr);
2556 cloneRegionBodyInto(partialDtorBlock, prev);
2557 builder.createYield(loc);
2558 });
2559 cir::YieldOp::create(builder, location: loc);
2560 });
2561 cir::YieldOp::create(builder&: b, location: loc);
2562 });
2563 } else {
2564 emitCtorDtorLoop();
2565 }
2566
2567 if (ifOp)
2568 cir::YieldOp::create(builder, location: loc);
2569
2570 op->erase();
2571}
2572
2573void LoweringPreparePass::lowerArrayDtor(cir::ArrayDtor op) {
2574 CIRBaseBuilderTy builder(getContext());
2575 builder.setInsertionPointAfter(op.getOperation());
2576
2577 mlir::Type eltTy = op->getRegion(index: 0).getArgument(i: 0).getType();
2578
2579 if (op.getNumElements()) {
2580 lowerArrayDtorCtorIntoLoop(builder, targetInfo: getTargetInfo(), op, eltTy,
2581 addr: op.getAddr(), numElements: op.getNumElements(),
2582 /*arrayLen=*/0,
2583 /*isCtor=*/false);
2584 return;
2585 }
2586
2587 auto arrayLen =
2588 mlir::cast<cir::ArrayType>(Val: op.getAddr().getType().getPointee()).getSize();
2589 lowerArrayDtorCtorIntoLoop(builder, targetInfo: getTargetInfo(), op, eltTy, addr: op.getAddr(),
2590 /*numElements=*/nullptr, arrayLen,
2591 /*isCtor=*/false);
2592}
2593
2594void LoweringPreparePass::lowerArrayCtor(cir::ArrayCtor op) {
2595 cir::CIRBaseBuilderTy builder(getContext());
2596 builder.setInsertionPointAfter(op.getOperation());
2597
2598 mlir::Type eltTy = op->getRegion(index: 0).getArgument(i: 0).getType();
2599
2600 if (op.getNumElements()) {
2601 lowerArrayDtorCtorIntoLoop(builder, targetInfo: getTargetInfo(), op, eltTy,
2602 addr: op.getAddr(), numElements: op.getNumElements(),
2603 /*arrayLen=*/0,
2604 /*isCtor=*/true);
2605 return;
2606 }
2607
2608 auto arrayLen =
2609 mlir::cast<cir::ArrayType>(Val: op.getAddr().getType().getPointee()).getSize();
2610 lowerArrayDtorCtorIntoLoop(builder, targetInfo: getTargetInfo(), op, eltTy, addr: op.getAddr(),
2611 /*numElements=*/nullptr, arrayLen,
2612 /*isCtor=*/true);
2613}
2614
2615cir::FuncOp LoweringPreparePass::getCalledFunction(cir::CallOp callOp) {
2616 mlir::SymbolRefAttr sym = llvm::dyn_cast_if_present<mlir::SymbolRefAttr>(
2617 Val: callOp.getCallableForCallee());
2618 if (!sym)
2619 return nullptr;
2620 return symbolTables.lookupNearestSymbolFrom<cir::FuncOp>(from: callOp, symbol: sym);
2621}
2622
2623void LoweringPreparePass::lowerTrivialCopyCall(cir::CallOp op) {
2624 cir::FuncOp funcOp = getCalledFunction(callOp: op);
2625 if (!funcOp)
2626 return;
2627
2628 std::optional<cir::CtorKind> ctorKind = funcOp.getCxxConstructorKind();
2629 if (ctorKind && *ctorKind == cir::CtorKind::Copy &&
2630 funcOp.isCxxTrivialMemberFunction()) {
2631 // Replace the trivial copy constructor call with a `CopyOp`
2632 CIRBaseBuilderTy builder(getContext());
2633 mlir::ValueRange operands = op.getOperands();
2634 mlir::Value dest = operands[0];
2635 mlir::Value src = operands[1];
2636 builder.setInsertionPoint(op);
2637 builder.createCopy(dst: dest, src);
2638 op.erase();
2639 }
2640}
2641
2642cir::GlobalOp LoweringPreparePass::getOrCreateConstAggregateGlobal(
2643 CIRBaseBuilderTy &builder, mlir::Location loc, llvm::StringRef baseName,
2644 mlir::Type ty, mlir::TypedAttr constant, uint64_t alignment) {
2645 // Look up (and lazily populate) the per-base-name cache.
2646 llvm::SmallVector<cir::GlobalOp, 1> &versions =
2647 constAggregateGlobals[baseName];
2648
2649 // First, check globals we've already discovered for this base name.
2650 for (cir::GlobalOp gv : versions) {
2651 if (gv.getSymType() == ty && gv.getInitialValue() == constant &&
2652 gv.getAlignment() == alignment)
2653 return gv;
2654 }
2655
2656 // No cached match. Scan the module's symbol table starting from the next
2657 // unscanned version. In practice this should usually exit on the first
2658 // iteration, but it's possible that some other pass or a previous
2659 // invocation of this pass created globals using this same logic.
2660 llvm::SmallString<128> name(baseName);
2661 size_t baseLen = name.size();
2662 unsigned version = versions.size();
2663 while (true) {
2664 name.resize(N: baseLen);
2665 if (version != 0) {
2666 name.push_back(Elt: '.');
2667 llvm::Twine(version).toVector(Out&: name);
2668 }
2669 auto existingGv = symbolTables.lookupSymbolIn<cir::GlobalOp>(
2670 symbolTableOp: mlirModule, name: mlir::StringAttr::get(context: &getContext(), bytes: name));
2671 if (!existingGv)
2672 break;
2673 versions.push_back(Elt: existingGv);
2674 if (existingGv.getSymType() == ty &&
2675 existingGv.getInitialValue() == constant &&
2676 existingGv.getAlignment() == alignment)
2677 return existingGv;
2678 ++version;
2679 }
2680
2681 // No match found, create a new global. The loop above found an unused name.
2682 mlir::OpBuilder::InsertionGuard guard(builder);
2683 builder.setInsertionPointToStart(mlirModule.getBody());
2684 auto gv =
2685 cir::GlobalOp::create(builder, location: loc, sym_name: name, sym_type: ty,
2686 /*isConstant=*/true,
2687 addrSpace: cir::LangAddressSpaceAttr::get(
2688 context: &getContext(), value: cir::LangAddressSpace::Default),
2689 linkage: cir::GlobalLinkageKind::PrivateLinkage);
2690 mlir::SymbolTable::setSymbolVisibility(
2691 symbol: gv, vis: mlir::SymbolTable::Visibility::Private);
2692 gv.setInitialValueAttr(constant);
2693 gv.setAlignment(alignment);
2694 // Matches classic codegen's createUnnamedGlobalFrom: only the contents of
2695 // the constant matter, not its address.
2696 gv.setUnnamedAddr(true);
2697
2698 // Keep the cached symbol table in sync with the new global so subsequent
2699 // lookups for other base names find it.
2700 symbolTables.getSymbolTable(op: mlirModule).insert(symbol: gv);
2701
2702 versions.push_back(Elt: gv);
2703 return gv;
2704}
2705
2706void LoweringPreparePass::lowerStoreOfConstAggregate(cir::StoreOp op) {
2707 // Check if the value operand is a cir.const with aggregate type.
2708 auto constOp = op.getValue().getDefiningOp<cir::ConstantOp>();
2709 if (!constOp)
2710 return;
2711
2712 mlir::Type ty = constOp.getType();
2713 if (!mlir::isa<cir::ArrayType, cir::RecordType>(Val: ty))
2714 return;
2715
2716 // Only transform stores to local variables (backed by cir.alloca).
2717 // Stores to other addresses (e.g. base_class_addr) should not be
2718 // transformed as they may be partial initializations.
2719 auto alloca = op.getAddr().getDefiningOp<cir::AllocaOp>();
2720 if (!alloca)
2721 return;
2722
2723 mlir::TypedAttr constant = constOp.getValue();
2724
2725 // A poison or undef aggregate has no bytes to copy, so there is nothing to
2726 // materialize in a global. CallConvLowering emits such stores for ignored
2727 // and empty-record arguments after this pass has run; leaving them alone
2728 // keeps a rerun of the pipeline from rewriting them into a copy.
2729 if (mlir::isa<cir::PoisonAttr, cir::UndefAttr>(Val: constant))
2730 return;
2731
2732 // OG implements several optimization tiers for constant aggregate
2733 // initialization. For now we always create a global constant + memcpy
2734 // (shouldCreateMemCpyFromGlobal). Future work can add the intermediate
2735 // tiers.
2736 assert(!cir::MissingFeatures::shouldUseBZeroPlusStoresToInitialize());
2737 assert(!cir::MissingFeatures::shouldUseMemSetToInitialize());
2738 assert(!cir::MissingFeatures::shouldSplitConstantStore());
2739
2740 // Get function name from parent cir.func.
2741 auto func = op->getParentOfType<cir::FuncOp>();
2742 if (!func)
2743 return;
2744 llvm::StringRef funcName = func.getSymName();
2745
2746 // Get variable name from the alloca.
2747 llvm::StringRef varName = alloca.getName();
2748
2749 // Build base name: __const.<func>.<var>
2750 std::string baseName = ("__const." + funcName + "." + varName).str();
2751 CIRBaseBuilderTy builder(getContext());
2752
2753 // Check for existing globals and create a new global with a unique name
2754 // if no match is found.
2755 cir::GlobalOp gv = getOrCreateConstAggregateGlobal(
2756 builder, loc: op.getLoc(), baseName, ty, constant, alignment: alloca.getAlignment());
2757
2758 // Now replace the store with get_global + copy.
2759 builder.setInsertionPoint(op);
2760
2761 auto ptrTy = cir::PointerType::get(pointee: ty);
2762 mlir::Value globalPtr =
2763 cir::GetGlobalOp::create(builder, location: op.getLoc(), addr: ptrTy, name: gv.getSymName());
2764
2765 // Replace store with copy.
2766 cir::CopyOp copyOp = builder.createCopy(dst: op.getAddr(), src: globalPtr);
2767
2768 cir::CIRDataLayout dataLayout(mlirModule);
2769 uint64_t naturalAlign = dataLayout.getTypeLanguageAlignment(ty);
2770 if (alloca.getAlignment() != naturalAlign)
2771 copyOp.setDstAlignment(alloca.getAlignment());
2772 uint64_t srcAlign = gv.getAlignment().value_or(u&: naturalAlign);
2773 if (srcAlign != naturalAlign)
2774 copyOp.setSrcAlignment(srcAlign);
2775
2776 // Erase the original store.
2777 op.erase();
2778
2779 // Erase the cir.const if it has no remaining users.
2780 if (constOp.use_empty())
2781 constOp.erase();
2782}
2783
2784// Every raised operation carries the original callee, the operands, and the
2785// attributes of the call, so this one function lowers any of them back to an
2786// equivalent plain call.
2787void LoweringPreparePass::lowerStdOp(cir::StdOpInterface typedOp) {
2788 mlir::Operation *op = typedOp.getOperation();
2789 cir::CIRBaseBuilderTy builder(getContext());
2790 builder.setInsertionPointAfter(op);
2791 mlir::Type resultType;
2792 if (op->getNumResults())
2793 resultType = op->getResult(idx: 0).getType();
2794 cir::CallOp call = builder.createCallOp(
2795 loc: op->getLoc(), callee: typedOp.getOriginalFnAttr(), returnType: resultType, operands: op->getOperands());
2796
2797 // IdiomRecognizer stores both the inherent and discardable attributes of the
2798 // original call as discardable attributes on the raised operation because
2799 // the cir.std operations do not share CallOp's property schema. Reconstruct
2800 // the original storage class from the destination CallOp schema here.
2801 //
2802 // This intentionally uses getInherentAttr as a schema query, not as a test
2803 // that `call` currently has the attribute: generated property accessors
2804 // return an engaged optional with a null attribute for a recognized but unset
2805 // optional property. An empty optional means the name is not inherent to
2806 // CallOp and must remain discardable.
2807 // TODO: Replace this transport encoding with shared call properties on the
2808 // raised operations.
2809 for (mlir::NamedAttribute attr : op->getDiscardableAttrs()) {
2810 if (call->getInherentAttr(name: attr.getName()).has_value())
2811 call->setInherentAttr(name: attr.getName(), value: attr.getValue());
2812 else
2813 call->setDiscardableAttr(name: attr.getName(), value: attr.getValue());
2814 }
2815
2816 op->replaceAllUsesWith(values&: call);
2817 op->erase();
2818}
2819
2820void LoweringPreparePass::runOnOp(mlir::Operation *op) {
2821 if (auto arrayCtor = dyn_cast<cir::ArrayCtor>(Val: op)) {
2822 lowerArrayCtor(op: arrayCtor);
2823 } else if (auto arrayDtor = dyn_cast<cir::ArrayDtor>(Val: op)) {
2824 lowerArrayDtor(op: arrayDtor);
2825 } else if (auto registerOp = dyn_cast<cir::RegisterExitDtorOp>(Val: op)) {
2826 lowerRegisterExitDtorOp(op: registerOp);
2827 } else if (auto stdOp = mlir::dyn_cast<cir::StdOpInterface>(Val: op)) {
2828 lowerStdOp(typedOp: stdOp);
2829 } else if (auto cast = mlir::dyn_cast<cir::CastOp>(Val: op)) {
2830 lowerCastOp(op: cast);
2831 } else if (auto complexConj = mlir::dyn_cast<cir::ComplexConjOp>(Val: op)) {
2832 lowerComplexConjOp(op: complexConj);
2833 } else if (auto complexDiv = mlir::dyn_cast<cir::ComplexFDivOp>(Val: op)) {
2834 lowerComplexFDivOp(op: complexDiv);
2835 } else if (auto complexMul = mlir::dyn_cast<cir::ComplexFMulOp>(Val: op)) {
2836 lowerComplexFMulOp(op: complexMul);
2837 } else if (auto complexDiv = mlir::dyn_cast<cir::ComplexDivOp>(Val: op)) {
2838 lowerComplexDivOp(op: complexDiv);
2839 } else if (auto complexMul = mlir::dyn_cast<cir::ComplexMulOp>(Val: op)) {
2840 lowerComplexMulOp(op: complexMul);
2841 } else if (auto glob = mlir::dyn_cast<cir::GlobalOp>(Val: op)) {
2842 lowerGlobalOp(op: glob);
2843 if (auto regAttr = glob->getAttrOfType<CUDAVarRegistrationInfoAttr>(
2844 name: CUDAVarRegistrationInfoAttr::getMnemonic()))
2845 cudaDeviceVars.emplace_back(Args&: glob, Args&: regAttr);
2846 } else if (auto getGlob = mlir::dyn_cast<cir::GetGlobalOp>(Val: op)) {
2847 lowerGetGlobalOp(op: getGlob);
2848 } else if (auto callOp = dyn_cast<cir::CallOp>(Val: op)) {
2849 lowerTrivialCopyCall(op: callOp);
2850 } else if (auto storeOp = dyn_cast<cir::StoreOp>(Val: op)) {
2851 lowerStoreOfConstAggregate(op: storeOp);
2852 } else if (auto fnOp = dyn_cast<cir::FuncOp>(Val: op)) {
2853 if (auto globalCtor = fnOp.getGlobalCtorPriority())
2854 globalCtorList.emplace_back(Args: fnOp.getName(), Args&: globalCtor.value());
2855 else if (auto globalDtor = fnOp.getGlobalDtorPriority())
2856 globalDtorList.emplace_back(Args: fnOp.getName(), Args&: globalDtor.value());
2857
2858 if (mlir::Attribute attr =
2859 fnOp->getAttr(name: cir::CUDAKernelNameAttr::getMnemonic());
2860 attr && !fnOp.isDeclaration()) {
2861 auto kernelNameAttr = dyn_cast<CUDAKernelNameAttr>(Val&: attr);
2862 llvm::StringRef kernelName = kernelNameAttr.getKernelName();
2863 cudaKernelMap[kernelName] = fnOp;
2864 }
2865 } else if (auto threeWayCmp = dyn_cast<cir::CmpThreeWayOp>(Val: op)) {
2866 lowerThreeWayCmpOp(op: threeWayCmp);
2867 } else if (auto initOp = dyn_cast<cir::LocalInitOp>(Val: op)) {
2868 lowerLocalInitOp(initOp);
2869 }
2870}
2871
2872static llvm::StringRef getCUDAPrefix(const clang::LangOptions &langOpts) {
2873 if (langOpts.HIP)
2874 return "hip";
2875 return "cuda";
2876}
2877
2878static std::string addUnderscoredPrefix(llvm::StringRef prefix,
2879 llvm::StringRef name) {
2880 return ("__" + prefix + name).str();
2881}
2882
2883/// Creates a global constructor function for the module:
2884///
2885/// For CUDA:
2886/// \code
2887/// void __cuda_module_ctor() {
2888/// Handle = __cudaRegisterFatBinary(GpuBinaryBlob);
2889/// __cuda_register_globals(Handle);
2890/// }
2891/// \endcode
2892///
2893/// For HIP:
2894/// \code
2895/// void __hip_module_ctor() {
2896/// if (__hip_gpubin_handle == 0) {
2897/// __hip_gpubin_handle = __hipRegisterFatBinary(GpuBinaryBlob);
2898/// __hip_register_globals(__hip_gpubin_handle);
2899/// }
2900/// }
2901/// \endcode
2902void LoweringPreparePass::buildCUDAModuleCtor() {
2903 bool isHIP = getLangOpts().HIP;
2904
2905 if (getLangOpts().GPURelocatableDeviceCode)
2906 llvm_unreachable("GPU RDC NYI");
2907
2908 // For CUDA without -fgpu-rdc, it's safe to stop generating ctor
2909 // if there's nothing to register.
2910 if (cudaKernelMap.empty() && cudaDeviceVars.empty())
2911 return;
2912
2913 // There's no device-side binary, so no need to proceed for CUDA.
2914 // HIP creates an external symbol in this case.
2915 auto deviceBinaryAttr = mlirModule->getAttrOfType<mlir::StringAttr>(
2916 name: CIRDialect::getCUDADeviceBinaryAttrName());
2917 if (!deviceBinaryAttr && !isHIP)
2918 return;
2919
2920 // Set up common types and builder.
2921 llvm::StringRef cudaPrefix = getCUDAPrefix(langOpts: getLangOpts());
2922 mlir::Location loc = mlirModule->getLoc();
2923 CIRBaseBuilderTy builder(getContext());
2924 builder.setInsertionPointToStart(mlirModule.getBody());
2925
2926 Type voidTy = builder.getVoidTy();
2927 PointerType voidPtrTy = builder.getVoidPtrTy();
2928 PointerType voidPtrPtrTy = builder.getPointerTo(ty: voidPtrTy);
2929 IntType intTy = builder.getSIntNTy(n: 32);
2930
2931 // --- Create fatbin globals ---
2932
2933 // The section names are different for MAC OS X.
2934 llvm::StringRef fatbinConstName =
2935 getLangOpts().HIP ? ".hip_fatbin" : ".nv_fatbin";
2936
2937 llvm::StringRef fatbinSectionName =
2938 getLangOpts().HIP ? ".hipFatBinSegment" : ".nvFatBinSegment";
2939
2940 GlobalOp fatbinStr;
2941 if (deviceBinaryAttr) {
2942 // Create the fatbin string constant with GPU binary contents.
2943 // The dialect verifier guarantees the attribute is typed as the array.
2944 auto fatbinType = mlir::cast<ArrayType>(Val: deviceBinaryAttr.getType());
2945 std::string fatbinStrName = addUnderscoredPrefix(prefix: cudaPrefix, name: "_fatbin_str");
2946 fatbinStr = GlobalOp::create(builder, location: loc, sym_name: fatbinStrName, sym_type: fatbinType,
2947 /*isConstant=*/true, addrSpace: {},
2948 linkage: GlobalLinkageKind::PrivateLinkage);
2949 constexpr unsigned HIPCodeObjectAlign = 4096;
2950 fatbinStr.setAlignment(isHIP ? HIPCodeObjectAlign : 8);
2951
2952 fatbinStr.setInitialValueAttr(
2953 cir::ConstArrayAttr::get(type: fatbinType, elts: deviceBinaryAttr));
2954 } else {
2955 // If fatbin is not available, create an external symbol __hip_fatbin in
2956 // section .hip_fatbin. It is populated elsewhere, e.g. by lld through a
2957 // linker script.
2958 fatbinStr = GlobalOp::create(
2959 builder, location: loc, sym_name: addCUIDSuffix(name: "__hip_fatbin"), sym_type: builder.getUIntNTy(n: 8),
2960 /*isConstant=*/true, addrSpace: {}, linkage: GlobalLinkageKind::ExternalLinkage);
2961 }
2962 fatbinStr.setSection(fatbinConstName);
2963 fatbinStr.setPrivate();
2964
2965 // Create the fatbin wrapper struct:
2966 // struct { int magic; int version; void *fatbin; void *unused; };
2967 mlir::Type fatbinWrapperMembers[] = {intTy, intTy, voidPtrTy, voidPtrTy};
2968 auto fatbinWrapperType = cir::StructType::get(
2969 context: &getContext(), members: fatbinWrapperMembers, /*packed=*/false, /*is_class=*/false,
2970 member_kinds: cir::RecordType::getAllDataKinds(members: fatbinWrapperMembers));
2971 std::string fatbinWrapperName =
2972 addUnderscoredPrefix(prefix: cudaPrefix, name: "_fatbin_wrapper");
2973 GlobalOp fatbinWrapper = GlobalOp::create(
2974 builder, location: loc, sym_name: fatbinWrapperName, sym_type: fatbinWrapperType,
2975 /*isConstant=*/true, addrSpace: {}, linkage: GlobalLinkageKind::PrivateLinkage);
2976 fatbinWrapper.setSection(fatbinSectionName);
2977
2978 constexpr unsigned cudaFatMagic = 0x466243b1;
2979 constexpr unsigned hipFatMagic = 0x48495046;
2980 unsigned fatMagic = isHIP ? hipFatMagic : cudaFatMagic;
2981
2982 auto magicInit = IntAttr::get(type: intTy, value: fatMagic);
2983 auto versionInit = IntAttr::get(type: intTy, value: 1);
2984 auto fatbinStrSymbol =
2985 mlir::FlatSymbolRefAttr::get(value: fatbinStr.getSymNameAttr());
2986 auto fatbinInit = GlobalViewAttr::get(type: voidPtrTy, symbol: fatbinStrSymbol);
2987 mlir::TypedAttr unusedInit = builder.getConstNullPtrAttr(t: voidPtrTy);
2988 fatbinWrapper.setInitialValueAttr(cir::ConstRecordAttr::get(
2989 type: fatbinWrapperType,
2990 members: mlir::ArrayAttr::get(context: &getContext(),
2991 value: {magicInit, versionInit, fatbinInit, unusedInit})));
2992
2993 // Create the GPU binary handle global variable.
2994 std::string gpubinHandleName =
2995 addUnderscoredPrefix(prefix: cudaPrefix, name: "_gpubin_handle");
2996 if (isHIP)
2997 gpubinHandleName = addCUIDSuffix(name: gpubinHandleName);
2998
2999 GlobalOp gpuBinHandle = GlobalOp::create(
3000 builder, location: loc, sym_name: gpubinHandleName, sym_type: voidPtrPtrTy,
3001 /*isConstant=*/false, addrSpace: {}, linkage: cir::GlobalLinkageKind::InternalLinkage);
3002 gpuBinHandle.setInitialValueAttr(builder.getConstNullPtrAttr(t: voidPtrPtrTy));
3003 gpuBinHandle.setPrivate();
3004
3005 // Declare this function:
3006 // void **__{cuda|hip}RegisterFatBinary(void *);
3007
3008 std::string regFuncName =
3009 addUnderscoredPrefix(prefix: cudaPrefix, name: "RegisterFatBinary");
3010 FuncType regFuncType = FuncType::get(inputs: {voidPtrTy}, returnType: voidPtrPtrTy);
3011 cir::FuncOp regFunc =
3012 buildRuntimeFunction(builder, name: regFuncName, loc, type: regFuncType);
3013
3014 std::string moduleCtorName = addUnderscoredPrefix(prefix: cudaPrefix, name: "_module_ctor");
3015 cir::FuncOp moduleCtor = buildRuntimeFunction(
3016 builder, name: moduleCtorName, loc, type: FuncType::get(inputs: {}, returnType: voidTy),
3017 linkage: GlobalLinkageKind::InternalLinkage);
3018
3019 globalCtorList.emplace_back(Args&: moduleCtorName,
3020 Args: cir::GlobalCtorAttr::getDefaultPriority());
3021 builder.setInsertionPointToStart(moduleCtor.addEntryBlock());
3022 assert(!cir::MissingFeatures::opGlobalCtorPriority());
3023 if (isHIP) {
3024 // --- Create HIP CTOR ---
3025 // if (__hip_gpubin_handle == nullptr)
3026 // __hip_gpubin_handle = __hipRegisterFatBinary(&fatbinWrapper);
3027 // __hip_register_globals(__hip_gpubin_handle);
3028 // atexit(__hip_module_dtor);
3029 mlir::Block *entryBlock = builder.getInsertionBlock();
3030 mlir::Region *parent = entryBlock->getParent();
3031 mlir::Block *ifBlock = builder.createBlock(parent);
3032 mlir::Block *exitBlock = builder.createBlock(parent);
3033 {
3034 mlir::OpBuilder::InsertionGuard guard(builder);
3035 builder.setInsertionPointToEnd(entryBlock);
3036 mlir::Value handle =
3037 builder.createLoad(loc, ptr: builder.createGetGlobal(global: gpuBinHandle));
3038 auto handlePtrTy = mlir::cast<cir::PointerType>(Val: handle.getType());
3039 mlir::Value nullPtr = builder.getNullPtr(ty: handlePtrTy, loc);
3040 mlir::Value isNull =
3041 builder.createCompare(loc, kind: cir::CmpOpKind::eq, lhs: handle, rhs: nullPtr);
3042 cir::BrCondOp::create(builder, location: loc, cond: isNull, destTrue: ifBlock, destFalse: exitBlock);
3043 }
3044 {
3045 // Handle is null: load the fatbin and register it.
3046 mlir::OpBuilder::InsertionGuard guard(builder);
3047 builder.setInsertionPointToStart(ifBlock);
3048 mlir::Value wrapper = builder.createGetGlobal(global: fatbinWrapper);
3049 mlir::Value fatbinVoidPtr = builder.createBitcast(src: wrapper, newTy: voidPtrTy);
3050 cir::CallOp gpuBinaryHandleCall =
3051 builder.createCallOp(loc, callee: regFunc, operands: fatbinVoidPtr);
3052 mlir::Value gpuBinaryHandle = gpuBinaryHandleCall.getResult();
3053 // Store the value back to the global `__hip_gpubin_handle`.
3054 mlir::Value gpuBinaryHandleGlobal = builder.createGetGlobal(global: gpuBinHandle);
3055 builder.createStore(loc, val: gpuBinaryHandle, dst: gpuBinaryHandleGlobal);
3056 cir::BrOp::create(builder, location: loc, dest: exitBlock);
3057 }
3058 {
3059 // Exit block: load the (possibly newly-registered) handle, call
3060 // __hip_register_globals, and register the module dtor with atexit().
3061 mlir::OpBuilder::InsertionGuard guard(builder);
3062 builder.setInsertionPointToStart(exitBlock);
3063 mlir::Value gHandle =
3064 builder.createLoad(loc, ptr: builder.createGetGlobal(global: gpuBinHandle));
3065
3066 if (std::optional<FuncOp> regGlobal = buildCUDARegisterGlobals())
3067 builder.createCallOp(loc, callee: *regGlobal, operands: gHandle);
3068
3069 if (std::optional<FuncOp> dtor = buildHIPModuleDtor()) {
3070 cir::CIRBaseBuilderTy globalBuilder(getContext());
3071 globalBuilder.setInsertionPointToStart(mlirModule.getBody());
3072 FuncOp atexit = buildRuntimeFunction(
3073 builder&: globalBuilder, name: "atexit", loc,
3074 type: FuncType::get(inputs: PointerType::get(pointee: dtor->getFunctionType()), returnType: intTy));
3075 mlir::Value dtorFunc = GetGlobalOp::create(
3076 builder, location: loc, addr: PointerType::get(pointee: dtor->getFunctionType()),
3077 name: mlir::FlatSymbolRefAttr::get(value: dtor->getSymNameAttr()));
3078 builder.createCallOp(loc, callee: atexit, operands: dtorFunc);
3079 }
3080 cir::ReturnOp::create(builder, location: loc);
3081 }
3082 return;
3083 }
3084 if (!getLangOpts().GPURelocatableDeviceCode) {
3085
3086 // --- Create CUDA CTOR-DTOR ---
3087 // Register binary with CUDA runtime. This is substantially different in
3088 // default mode vs. separate compilation.
3089 // Corresponding code:
3090 // gpuBinaryHandle = __cudaRegisterFatBinary(&fatbinWrapper);
3091 mlir::Value wrapper = builder.createGetGlobal(global: fatbinWrapper);
3092 mlir::Value fatbinVoidPtr = builder.createBitcast(src: wrapper, newTy: voidPtrTy);
3093 cir::CallOp gpuBinaryHandleCall =
3094 builder.createCallOp(loc, callee: regFunc, operands: fatbinVoidPtr);
3095 mlir::Value gpuBinaryHandle = gpuBinaryHandleCall.getResult();
3096 // Store the value back to the global `__cuda_gpubin_handle`.
3097 mlir::Value gpuBinaryHandleGlobal = builder.createGetGlobal(global: gpuBinHandle);
3098 builder.createStore(loc, val: gpuBinaryHandle, dst: gpuBinaryHandleGlobal);
3099
3100 // --- Generate __cuda_register_globals and call it ---
3101 if (std::optional<FuncOp> regGlobal = buildCUDARegisterGlobals()) {
3102 builder.createCallOp(loc, callee: *regGlobal, operands: gpuBinaryHandle);
3103 }
3104
3105 // From CUDA 10.1 onwards, we must call this function to end registration:
3106 // void __cudaRegisterFatBinaryEnd(void **fatbinHandle);
3107 // This is CUDA-specific, so no need to use `addUnderscoredPrefix`.
3108 std::optional<llvm::VersionTuple> sdkVersion = getSDKVersion();
3109 if (!sdkVersion)
3110 return;
3111 if (clang::CudaFeatureEnabled(
3112 *sdkVersion, clang::CudaFeature::CUDA_USES_FATBIN_REGISTER_END)) {
3113 cir::CIRBaseBuilderTy globalBuilder(getContext());
3114 globalBuilder.setInsertionPointToStart(mlirModule.getBody());
3115 FuncOp endFunc =
3116 buildRuntimeFunction(builder&: globalBuilder, name: "__cudaRegisterFatBinaryEnd", loc,
3117 type: FuncType::get(inputs: {voidPtrPtrTy}, returnType: voidTy));
3118 builder.createCallOp(loc, callee: endFunc, operands: gpuBinaryHandle);
3119 }
3120 } else
3121 llvm_unreachable("GPU RDC NYI");
3122
3123 // Create destructor and register it with atexit() the way NVCC does it. Doing
3124 // it during regular destructor phase worked in CUDA before 9.2 but results in
3125 // double-free in 9.2.
3126 if (std::optional<FuncOp> dtor = buildCUDAModuleDtor()) {
3127
3128 // extern "C" int atexit(void (*f)(void));
3129 cir::CIRBaseBuilderTy globalBuilder(getContext());
3130 globalBuilder.setInsertionPointToStart(mlirModule.getBody());
3131 FuncOp atexit = buildRuntimeFunction(
3132 builder&: globalBuilder, name: "atexit", loc,
3133 type: FuncType::get(inputs: PointerType::get(pointee: dtor->getFunctionType()), returnType: intTy));
3134 mlir::Value dtorFunc = GetGlobalOp::create(
3135 builder, location: loc, addr: PointerType::get(pointee: dtor->getFunctionType()),
3136 name: mlir::FlatSymbolRefAttr::get(value: dtor->getSymNameAttr()));
3137 builder.createCallOp(loc, callee: atexit, operands: dtorFunc);
3138 }
3139 cir::ReturnOp::create(builder, location: loc);
3140}
3141
3142std::optional<FuncOp> LoweringPreparePass::buildCUDAModuleDtor() {
3143 if (!mlirModule->getAttr(name: CIRDialect::getCUDADeviceBinaryAttrName()))
3144 return {};
3145
3146 llvm::StringRef prefix = getCUDAPrefix(langOpts: getLangOpts());
3147
3148 VoidType voidTy = VoidType::get(ctx: &getContext());
3149 PointerType voidPtrPtrTy = PointerType::get(pointee: PointerType::get(pointee: voidTy));
3150
3151 mlir::Location loc = mlirModule.getLoc();
3152
3153 cir::CIRBaseBuilderTy builder(getContext());
3154 builder.setInsertionPointToStart(mlirModule.getBody());
3155
3156 // define: void __cudaUnregisterFatBinary(void ** handle);
3157 std::string unregisterFuncName =
3158 addUnderscoredPrefix(prefix, name: "UnregisterFatBinary");
3159 FuncOp unregisterFunc = buildRuntimeFunction(
3160 builder, name: unregisterFuncName, loc, type: FuncType::get(inputs: {voidPtrPtrTy}, returnType: voidTy));
3161
3162 // void __cuda_module_dtor();
3163 // Despite the name, OG doesn't treat it as a destructor, so it shouldn't be
3164 // put into globalDtorList. If it were a real dtor, then it would cause
3165 // double free above CUDA 9.2. The way to use it is to manually call
3166 // atexit() at end of module ctor.
3167 std::string dtorName = addUnderscoredPrefix(prefix, name: "_module_dtor");
3168 FuncOp dtor =
3169 buildRuntimeFunction(builder, name: dtorName, loc, type: FuncType::get(inputs: {}, returnType: voidTy),
3170 linkage: GlobalLinkageKind::InternalLinkage);
3171
3172 builder.setInsertionPointToStart(dtor.addEntryBlock());
3173
3174 // For dtor, we only need to call:
3175 // __cudaUnregisterFatBinary(__cuda_gpubin_handle);
3176
3177 std::string gpubinName = addUnderscoredPrefix(prefix, name: "_gpubin_handle");
3178 GlobalOp gpubinGlobal = cast<GlobalOp>(Val: mlirModule.lookupSymbol(name: gpubinName));
3179 mlir::Value gpubinAddress = builder.createGetGlobal(global: gpubinGlobal);
3180 mlir::Value gpubin = builder.createLoad(loc, ptr: gpubinAddress);
3181 builder.createCallOp(loc, callee: unregisterFunc, operands: gpubin);
3182 ReturnOp::create(builder, location: loc);
3183
3184 return dtor;
3185}
3186
3187/// Build the HIP module dtor:
3188///
3189/// void __hip_module_dtor() {
3190/// if (__hip_gpubin_handle != nullptr) {
3191/// __hipUnregisterFatBinary(__hip_gpubin_handle);
3192/// __hip_gpubin_handle = nullptr;
3193/// }
3194/// }
3195///
3196/// Despite the name, OG doesn't treat this as a real destructor: putting it on
3197/// the dtor list would cause a double-free. It is meant to be registered via
3198/// atexit() at the end of the module ctor.
3199std::optional<FuncOp> LoweringPreparePass::buildHIPModuleDtor() {
3200 llvm::StringRef prefix = getCUDAPrefix(langOpts: getLangOpts());
3201
3202 VoidType voidTy = VoidType::get(ctx: &getContext());
3203 PointerType voidPtrPtrTy = PointerType::get(pointee: PointerType::get(pointee: voidTy));
3204
3205 mlir::Location loc = mlirModule.getLoc();
3206
3207 cir::CIRBaseBuilderTy builder(getContext());
3208 builder.setInsertionPointToStart(mlirModule.getBody());
3209
3210 // void __hipUnregisterFatBinary(void ** handle);
3211 std::string unregisterFuncName =
3212 addUnderscoredPrefix(prefix, name: "UnregisterFatBinary");
3213 FuncOp unregisterFunc = buildRuntimeFunction(
3214 builder, name: unregisterFuncName, loc, type: FuncType::get(inputs: {voidPtrPtrTy}, returnType: voidTy));
3215
3216 std::string dtorName = addUnderscoredPrefix(prefix, name: "_module_dtor");
3217 FuncOp dtor =
3218 buildRuntimeFunction(builder, name: dtorName, loc, type: FuncType::get(inputs: {}, returnType: voidTy),
3219 linkage: GlobalLinkageKind::InternalLinkage);
3220
3221 std::string gpubinName =
3222 addCUIDSuffix(name: addUnderscoredPrefix(prefix, name: "_gpubin_handle"));
3223 GlobalOp gpuBinGlobal = cast<GlobalOp>(Val: mlirModule.lookupSymbol(name: gpubinName));
3224
3225 mlir::Block *entryBlock = dtor.addEntryBlock();
3226 mlir::Block *ifBlock = builder.createBlock(parent: &dtor.getBody());
3227 mlir::Block *exitBlock = builder.createBlock(parent: &dtor.getBody());
3228
3229 mlir::OpBuilder::InsertionGuard guard(builder);
3230 builder.setInsertionPointToEnd(entryBlock);
3231 mlir::Value handle =
3232 builder.createLoad(loc, ptr: builder.createGetGlobal(global: gpuBinGlobal));
3233 auto handlePtrTy = mlir::cast<cir::PointerType>(Val: handle.getType());
3234 mlir::Value nullPtr = builder.getNullPtr(ty: handlePtrTy, loc);
3235 mlir::Value isNotNull =
3236 builder.createCompare(loc, kind: cir::CmpOpKind::ne, lhs: handle, rhs: nullPtr);
3237 cir::BrCondOp::create(builder, location: loc, cond: isNotNull, destTrue: ifBlock, destFalse: exitBlock);
3238
3239 {
3240 // Handle is non-null: unregister and clear it.
3241 mlir::OpBuilder::InsertionGuard ifGuard(builder);
3242 builder.setInsertionPointToStart(ifBlock);
3243 builder.createCallOp(loc, callee: unregisterFunc, operands: handle);
3244 builder.createStore(loc, val: nullPtr, dst: builder.createGetGlobal(global: gpuBinGlobal));
3245 cir::BrOp::create(builder, location: loc, dest: exitBlock);
3246 }
3247 {
3248 mlir::OpBuilder::InsertionGuard exitGuard(builder);
3249 builder.setInsertionPointToStart(exitBlock);
3250 cir::ReturnOp::create(builder, location: loc);
3251 }
3252
3253 return dtor;
3254}
3255
3256std::optional<FuncOp> LoweringPreparePass::buildCUDARegisterGlobals() {
3257 if (cudaKernelMap.empty() && cudaDeviceVars.empty())
3258 return {};
3259
3260 cir::CIRBaseBuilderTy builder(getContext());
3261 builder.setInsertionPointToStart(mlirModule.getBody());
3262
3263 mlir::Location loc = mlirModule.getLoc();
3264 llvm::StringRef cudaPrefix = getCUDAPrefix(langOpts: getLangOpts());
3265
3266 auto voidTy = VoidType::get(ctx: &getContext());
3267 auto voidPtrTy = PointerType::get(pointee: voidTy);
3268 auto voidPtrPtrTy = PointerType::get(pointee: voidPtrTy);
3269
3270 // Create the function:
3271 // void __cuda_register_globals(void **fatbinHandle)
3272 std::string regGlobalFuncName =
3273 addUnderscoredPrefix(prefix: cudaPrefix, name: "_register_globals");
3274 auto regGlobalFuncTy = FuncType::get(inputs: {voidPtrPtrTy}, returnType: voidTy);
3275 FuncOp regGlobalFunc =
3276 buildRuntimeFunction(builder, name: regGlobalFuncName, loc, type: regGlobalFuncTy,
3277 /*linkage=*/GlobalLinkageKind::InternalLinkage);
3278 builder.setInsertionPointToStart(regGlobalFunc.addEntryBlock());
3279
3280 buildCUDARegisterGlobalFunctions(builder, regGlobalFunc);
3281 buildCUDARegisterVars(builder, regGlobalFunc);
3282
3283 ReturnOp::create(builder, location: loc);
3284 return regGlobalFunc;
3285}
3286
3287void LoweringPreparePass::buildCUDARegisterGlobalFunctions(
3288 cir::CIRBaseBuilderTy &builder, FuncOp regGlobalFunc) {
3289 mlir::Location loc = mlirModule.getLoc();
3290 llvm::StringRef cudaPrefix = getCUDAPrefix(langOpts: getLangOpts());
3291 cir::CIRDataLayout dataLayout(mlirModule);
3292
3293 auto voidTy = VoidType::get(ctx: &getContext());
3294 auto voidPtrTy = PointerType::get(pointee: voidTy);
3295 auto voidPtrPtrTy = PointerType::get(pointee: voidPtrTy);
3296 IntType intTy = builder.getSIntNTy(n: 32);
3297 IntType charTy =
3298 cir::IntType::get(context: &getContext(), width: getTargetInfo().getCharWidth(),
3299 /*isSigned=*/is_signed: false);
3300
3301 // Extract the GPU binary handle argument.
3302 mlir::Value fatbinHandle = *regGlobalFunc.args_begin();
3303
3304 cir::CIRBaseBuilderTy globalBuilder(getContext());
3305 globalBuilder.setInsertionPointToStart(mlirModule.getBody());
3306
3307 // Declare CUDA internal functions:
3308 // int __cudaRegisterFunction(
3309 // void **fatbinHandle,
3310 // const char *hostFunc,
3311 // char *deviceFunc,
3312 // const char *deviceName,
3313 // int threadLimit,
3314 // uint3 *tid, uint3 *bid, dim3 *bDim, dim3 *gDim,
3315 // int *wsize
3316 // )
3317 // OG doesn't care about the types at all. They're treated as void*.
3318
3319 FuncOp cudaRegisterFunction = buildRuntimeFunction(
3320 builder&: globalBuilder, name: addUnderscoredPrefix(prefix: cudaPrefix, name: "RegisterFunction"), loc,
3321 type: FuncType::get(inputs: {voidPtrPtrTy, voidPtrTy, voidPtrTy, voidPtrTy, intTy,
3322 voidPtrTy, voidPtrTy, voidPtrTy, voidPtrTy, voidPtrTy},
3323 returnType: intTy));
3324
3325 auto makeConstantString = [&](llvm::StringRef str) -> GlobalOp {
3326 auto strType = ArrayType::get(context: &getContext(), elementType: charTy, size: 1 + str.size());
3327 auto tmpString = cir::GlobalOp::create(
3328 builder&: globalBuilder, location: loc, sym_name: (".str" + str).str(), sym_type: strType,
3329 /*isConstant=*/true, addrSpace: {},
3330 /*linkage=*/cir::GlobalLinkageKind::PrivateLinkage);
3331
3332 // We must make the string zero-terminated.
3333 tmpString.setInitialValueAttr(
3334 ConstArrayAttr::get(type: strType, elts: StringAttr::get(bytes: str + "\0", type: strType)));
3335 tmpString.setPrivate();
3336 return tmpString;
3337 };
3338
3339 cir::ConstantOp cirNullPtr = builder.getNullPtr(ty: voidPtrTy, loc);
3340 bool isHIP = getLangOpts().HIP;
3341 for (auto kernelName : cudaKernelMap.keys()) {
3342 FuncOp deviceStub = cudaKernelMap[kernelName];
3343 GlobalOp deviceFuncStr = makeConstantString(kernelName);
3344 mlir::Value deviceFunc = builder.createBitcast(
3345 src: builder.createGetGlobal(global: deviceFuncStr), newTy: voidPtrTy);
3346
3347 mlir::Value hostFunc;
3348 if (isHIP) {
3349 // Under HIP, the kernel-handle is a GlobalOp shadow created by CIR
3350 // codegen and named with the kernel-reference mangled name (e.g.
3351 // `@_Z2fnv` pointing at the device-stub function
3352 // `_Z17__device_stub__fnv`). The CUDAKernelNameAttr on the device-stub
3353 // uses the same name, so we can resolve the shadow by symbol lookup.
3354 auto funcHandle = cast<GlobalOp>(Val: mlirModule.lookupSymbol(name: kernelName));
3355 hostFunc =
3356 builder.createBitcast(src: builder.createGetGlobal(global: funcHandle), newTy: voidPtrTy);
3357 } else {
3358 hostFunc = builder.createBitcast(
3359 src: GetGlobalOp::create(
3360 builder, location: loc, addr: PointerType::get(pointee: deviceStub.getFunctionType()),
3361 name: mlir::FlatSymbolRefAttr::get(value: deviceStub.getSymNameAttr())),
3362 newTy: voidPtrTy);
3363 }
3364 builder.createCallOp(
3365 loc, callee: cudaRegisterFunction,
3366 operands: {fatbinHandle, hostFunc, deviceFunc, deviceFunc,
3367 ConstantOp::create(builder, location: loc, value: IntAttr::get(type: intTy, value: -1)), cirNullPtr,
3368 cirNullPtr, cirNullPtr, cirNullPtr, cirNullPtr});
3369 }
3370}
3371
3372// Emit `__{cuda|hip}RegisterVar` calls inside `__{cuda|hip}_register_globals`
3373// for every device-side shadow that carries a `cu.var_registration` attribute
3374// (attached by `CIRGenNVCUDARuntime::handleVarRegistration`).
3375void LoweringPreparePass::buildCUDARegisterVars(cir::CIRBaseBuilderTy &builder,
3376 FuncOp regGlobalFunc) {
3377 mlir::Location loc = mlirModule.getLoc();
3378 llvm::StringRef cudaPrefix = getCUDAPrefix(langOpts: getLangOpts());
3379 cir::CIRDataLayout dataLayout(mlirModule);
3380
3381 PointerType voidPtrTy = builder.getVoidPtrTy();
3382 PointerType voidPtrPtrTy = builder.getPointerTo(ty: voidPtrTy);
3383 IntType intTy = builder.getSIntNTy(n: 32);
3384 IntType sizeTy = builder.getUIntNTy(n: getTargetInfo().getMaxPointerWidth());
3385 IntType charTy =
3386 cir::IntType::get(context: &getContext(), width: getTargetInfo().getCharWidth(),
3387 /*isSigned=*/is_signed: false);
3388
3389 if (cudaDeviceVars.empty())
3390 return;
3391
3392 cir::CIRBaseBuilderTy globalBuilder(getContext());
3393 globalBuilder.setInsertionPointToStart(mlirModule.getBody());
3394
3395 cir::VoidType voidTy = builder.getVoidTy();
3396 FuncOp cudaRegisterVar;
3397 FuncOp cudaRegisterManagedVar;
3398
3399 // void __cudaRegisterSurface(void **fatbinHandle,
3400 // const struct surfaceReference *hostVar,
3401 // const void **deviceAddress,
3402 // const char *deviceName,
3403 // int type, int ext);
3404 FuncOp cudaRegisterSurface = buildRuntimeFunction(
3405 builder&: globalBuilder, name: addUnderscoredPrefix(prefix: cudaPrefix, name: "RegisterSurface"), loc,
3406 type: FuncType::get(
3407 inputs: {voidPtrPtrTy, voidPtrTy, voidPtrTy, voidPtrTy, intTy, intTy},
3408 returnType: voidTy));
3409
3410 // void __cudaRegisterTexture(void **fatbinHandle,
3411 // const struct textureReference *hostVar,
3412 // const void **deviceAddress,
3413 // const char *deviceName,
3414 // int textureType, int normalized, int ext);
3415 FuncOp cudaRegisterTexture = buildRuntimeFunction(
3416 builder&: globalBuilder, name: addUnderscoredPrefix(prefix: cudaPrefix, name: "RegisterTexture"), loc,
3417 type: FuncType::get(
3418 inputs: {voidPtrPtrTy, voidPtrTy, voidPtrTy, voidPtrTy, intTy, intTy, intTy},
3419 returnType: voidTy));
3420
3421 auto makeConstantString = [&](llvm::StringRef str) -> GlobalOp {
3422 auto strType = ArrayType::get(context: &getContext(), elementType: charTy, size: 1 + str.size());
3423 auto tmpString = cir::GlobalOp::create(
3424 builder&: globalBuilder, location: loc, sym_name: (".str" + str).str(), sym_type: strType,
3425 /*isConstant=*/true, addrSpace: {},
3426 /*linkage=*/cir::GlobalLinkageKind::PrivateLinkage);
3427 tmpString.setInitialValueAttr(
3428 ConstArrayAttr::get(type: strType, elts: StringAttr::get(bytes: str + "\0", type: strType)));
3429 tmpString.setPrivate();
3430 return tmpString;
3431 };
3432
3433 mlir::Value fatbinHandle = *regGlobalFunc.args_begin();
3434
3435 for (auto &[global, regAttr] : cudaDeviceVars) {
3436 // External managed variables are registered by their defining TU.
3437 if (regAttr.getIsManaged() && global.isDeclaration())
3438 continue;
3439
3440 GlobalOp deviceNameStr = makeConstantString(regAttr.getDeviceSideName());
3441 mlir::Value deviceName = builder.createBitcast(
3442 src: builder.createGetGlobal(global: deviceNameStr), newTy: voidPtrTy);
3443 mlir::Value hostVar =
3444 builder.createBitcast(src: builder.createGetGlobal(global), newTy: voidPtrTy);
3445
3446 llvm::TypeSize size = dataLayout.getTypeAllocSize(ty: global.getSymType());
3447
3448 if (regAttr.getIsManaged()) {
3449 // void __hipRegisterManagedVar(void **fatbinHandle, char *managedVar,
3450 // char *var, const char *deviceName,
3451 // size_t size, unsigned align);
3452 if (!cudaRegisterManagedVar)
3453 cudaRegisterManagedVar = buildRuntimeFunction(
3454 builder&: globalBuilder,
3455 name: addUnderscoredPrefix(prefix: cudaPrefix, name: "RegisterManagedVar"), loc,
3456 type: FuncType::get(
3457 inputs: {voidPtrPtrTy, voidPtrTy, voidPtrTy, voidPtrTy, sizeTy, intTy},
3458 returnType: voidTy));
3459
3460 llvm::StringRef name = global.getSymName();
3461 assert(name.ends_with(".managed") &&
3462 "HIP managed variables not transformed");
3463 auto managedVar = mlirModule.lookupSymbol<GlobalOp>(
3464 name: name.drop_back(N: llvm::StringRef(".managed").size()));
3465 mlir::Value managedPtr =
3466 builder.createBitcast(src: builder.createGetGlobal(global: managedVar), newTy: voidPtrTy);
3467 auto varSize = ConstantOp::create(
3468 builder, location: loc, value: IntAttr::get(type: sizeTy, value: size.getFixedValue()));
3469 auto align = ConstantOp::create(
3470 builder, location: loc, value: IntAttr::get(type: intTy, value: global.getAlignment().value_or(u: 1)));
3471 builder.createCallOp(
3472 loc, callee: cudaRegisterManagedVar,
3473 operands: {fatbinHandle, managedPtr, hostVar, deviceName, varSize, align});
3474 } else {
3475 auto isExtern = ConstantOp::create(
3476 builder, location: loc, value: IntAttr::get(type: intTy, value: regAttr.getIsExtern() ? 1 : 0));
3477
3478 switch (regAttr.getKind()) {
3479 case cir::CUDADeviceVarKind::Variable: {
3480 // void __{cuda|hip}RegisterVar(void **fatbinHandle,
3481 // char *hostVar, char *deviceAddress,
3482 // const char *deviceName, int ext,
3483 // size_t size, int constant,
3484 // int normalized);
3485 // OG ignores parameter types, treating pointers as void*.
3486 if (!cudaRegisterVar)
3487 cudaRegisterVar = buildRuntimeFunction(
3488 builder&: globalBuilder, name: addUnderscoredPrefix(prefix: cudaPrefix, name: "RegisterVar"),
3489 loc,
3490 type: FuncType::get(inputs: {voidPtrPtrTy, voidPtrTy, voidPtrTy, voidPtrTy,
3491 intTy, sizeTy, intTy, intTy},
3492 returnType: voidTy));
3493
3494 auto varSize = ConstantOp::create(
3495 builder, location: loc, value: IntAttr::get(type: sizeTy, value: size.getFixedValue()));
3496 auto isConstant = ConstantOp::create(
3497 builder, location: loc, value: IntAttr::get(type: intTy, value: regAttr.getIsConstant() ? 1 : 0));
3498 auto normalized =
3499 ConstantOp::create(builder, location: loc, value: IntAttr::get(type: intTy, value: 0));
3500
3501 builder.createCallOp(loc, callee: cudaRegisterVar,
3502 operands: {fatbinHandle, hostVar, deviceName, deviceName,
3503 isExtern, varSize, isConstant, normalized});
3504 break;
3505 }
3506
3507 case cir::CUDADeviceVarKind::Surface: {
3508 cir::ConstantOp surfaceType = ConstantOp::create(
3509 builder, location: loc, value: IntAttr::get(type: intTy, value: regAttr.getSurfTexType()));
3510 builder.createCallOp(loc, callee: cudaRegisterSurface,
3511 operands: {fatbinHandle, hostVar, deviceName, deviceName,
3512 surfaceType, isExtern});
3513 break;
3514 }
3515
3516 case cir::CUDADeviceVarKind::Texture: {
3517 auto textureType = ConstantOp::create(
3518 builder, location: loc, value: IntAttr::get(type: intTy, value: regAttr.getSurfTexType()));
3519 auto normalized = ConstantOp::create(
3520 builder, location: loc, value: IntAttr::get(type: intTy, value: regAttr.getNormalized() ? 1 : 0));
3521
3522 builder.createCallOp(loc, callee: cudaRegisterTexture,
3523 operands: {fatbinHandle, hostVar, deviceName, deviceName,
3524 textureType, normalized, isExtern});
3525 break;
3526 }
3527 }
3528 }
3529 }
3530}
3531
3532void LoweringPreparePass::runOnOperation() {
3533 mlirModule = getOperation();
3534
3535 lowerModule = cir::createLowerModule(module: mlirModule);
3536 assert(lowerModule && "LowerModule shouldn't be null");
3537 if (!lowerModule->hasTarget()) {
3538 mlirModule.emitError() << getArgument()
3539 << " requires a module with a supported '"
3540 << CIRDialect::getTripleAttrName() << "' attribute";
3541 return signalPassFailure();
3542 }
3543
3544 llvm::SmallVector<mlir::Operation *> opsToTransform;
3545
3546 mlirModule->walk(callback: [&](mlir::Operation *op) {
3547 if (mlir::isa<cir::ArrayCtor, cir::ArrayDtor, cir::CastOp,
3548 cir::ComplexConjOp, cir::ComplexMulOp, cir::ComplexDivOp,
3549 cir::ComplexFMulOp, cir::ComplexFDivOp, cir::DynamicCastOp,
3550 cir::FuncOp, cir::CallOp, cir::GetGlobalOp, cir::GlobalOp,
3551 cir::StoreOp, cir::CmpThreeWayOp, cir::LocalInitOp,
3552 cir::RegisterExitDtorOp, cir::StdOpInterface>(Val: op))
3553 opsToTransform.push_back(Elt: op);
3554 });
3555
3556 for (mlir::Operation *o : opsToTransform)
3557 runOnOp(op: o);
3558
3559 // Every TLS wrapper, init alias and wrapper call has been built from
3560 // 'tls_refs' by now. Nothing after this pass reads it, so drop it to keep a
3561 // second run of this pass from building the wrappers again.
3562 for (auto globalOp : mlirModule.getOps<cir::GlobalOp>())
3563 globalOp.removeTlsRefsAttr();
3564
3565 buildCXXGlobalInitFunc();
3566 // The C++20 module initializer has been built from these. Nothing after this
3567 // pass reads them, so drop them to keep a second run of this pass from
3568 // building it again.
3569 mlirModule->removeAttr(name: CIRDialect::getCXXModuleInitFnNameAttrName());
3570 mlirModule->removeAttr(name: CIRDialect::getCXXModuleImportedInitsAttrName());
3571 buildCXXGlobalTlsFunc();
3572 if (getLangOpts().CUDA && !getLangOpts().CUDAIsDevice) {
3573 buildCUDAModuleCtor();
3574 // The fatbin global now references the same attribute; drop the module's
3575 // reference so an emitted .cir doesn't print the bytes twice. This has to
3576 // happen out here because the ctor and the CUDA dtor builder test the
3577 // attribute to decide whether a device-side binary exists at all.
3578 mlirModule->removeAttr(name: CIRDialect::getCUDADeviceBinaryAttrName());
3579 // The kernels and device variables have been registered from these.
3580 // Nothing after this pass reads them, so drop them to keep a second run
3581 // of this pass from registering them again.
3582 for (auto &entry : cudaKernelMap)
3583 entry.getValue()->removeAttr(name: CUDAKernelNameAttr::getMnemonic());
3584 for (auto &deviceVar : cudaDeviceVars)
3585 deviceVar.first->removeAttr(name: CUDAVarRegistrationInfoAttr::getMnemonic());
3586 }
3587
3588 buildGlobalCtorDtorList();
3589}
3590
3591std::unique_ptr<Pass> mlir::createLoweringPreparePass() {
3592 return std::make_unique<LoweringPreparePass>();
3593}
3594