1//===- CIRGenModule.cpp - Per-Module state for CIR generation -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This is the internal per-translation-unit state used for CIR translation.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CIRGenModule.h"
14#include "CIRGenCUDARuntime.h"
15#include "CIRGenCXXABI.h"
16#include "CIRGenConstantEmitter.h"
17#include "CIRGenFunction.h"
18
19#include "mlir/Dialect/LLVMIR/LLVMDialectDecl.h"
20#include "mlir/Dialect/OpenMP/Utils/Utils.h"
21#include "mlir/IR/SymbolTable.h"
22#include "clang/AST/ASTContext.h"
23#include "clang/AST/ASTLambda.h"
24#include "clang/AST/Attrs.inc"
25#include "clang/AST/DeclBase.h"
26#include "clang/AST/DeclOpenACC.h"
27#include "clang/AST/GlobalDecl.h"
28#include "clang/AST/Mangle.h"
29#include "clang/AST/RecordLayout.h"
30#include "clang/AST/StmtOpenMP.h"
31#include "clang/Basic/DiagnosticFrontend.h"
32#include "clang/Basic/Module.h"
33#include "clang/Basic/SourceManager.h"
34#include "clang/CIR/Dialect/IR/CIRAttrs.h"
35#include "clang/CIR/Dialect/IR/CIRDialect.h"
36#include "clang/CIR/Dialect/IR/CIROpsEnums.h"
37#include "clang/CIR/Dialect/IR/CIRTypes.h"
38#include "clang/CIR/Interfaces/CIROpInterfaces.h"
39#include "clang/CIR/MissingFeatures.h"
40#include "clang/CodeGenUtils/ModuleUtils.h"
41#include "clang/CodeGenUtils/RecordLayoutUtils.h"
42#include "llvm/ADT/STLExtras.h"
43#include "llvm/ADT/StringExtras.h"
44#include "llvm/ADT/StringRef.h"
45#include "llvm/ADT/StringSwitch.h"
46#include "llvm/Support/raw_ostream.h"
47
48#include "CIRGenFunctionInfo.h"
49#include "TargetInfo.h"
50#include "mlir/Dialect/Ptr/IR/MemorySpaceInterfaces.h"
51#include "mlir/IR/Attributes.h"
52#include "mlir/IR/BuiltinOps.h"
53#include "mlir/IR/Location.h"
54#include "mlir/IR/MLIRContext.h"
55#include "mlir/IR/Operation.h"
56#include "mlir/IR/Verifier.h"
57
58#include <algorithm>
59
60using namespace clang;
61using namespace clang::CIRGen;
62
63static CIRGenCXXABI *createCXXABI(CIRGenModule &cgm) {
64 switch (cgm.getASTContext().getCXXABIKind()) {
65 case TargetCXXABI::GenericItanium:
66 case TargetCXXABI::GenericAArch64:
67 case TargetCXXABI::AppleARM64:
68 case TargetCXXABI::GenericARM:
69 return CreateCIRGenItaniumCXXABI(cgm);
70 case TargetCXXABI::Microsoft:
71 return CreateCIRGenMicrosoftCXXABI(cgm);
72
73 case TargetCXXABI::Fuchsia:
74 case TargetCXXABI::iOS:
75 case TargetCXXABI::WatchOS:
76 case TargetCXXABI::GenericMIPS:
77 case TargetCXXABI::WebAssembly:
78 case TargetCXXABI::XL:
79 cgm.errorNYI(feature: "createCXXABI: C++ ABI kind");
80 return nullptr;
81 }
82
83 llvm_unreachable("invalid C++ ABI kind");
84}
85
86CIRGenModule::CIRGenModule(mlir::MLIRContext &mlirContext,
87 clang::ASTContext &astContext,
88 const clang::CodeGenOptions &cgo,
89 DiagnosticsEngine &diags)
90 : builder(mlirContext, *this), astContext(astContext),
91 langOpts(astContext.getLangOpts()), atomicOpts(astContext.getLangOpts()),
92 codeGenOpts(cgo),
93 theModule{mlir::ModuleOp::create(loc: mlir::UnknownLoc::get(context: &mlirContext))},
94 diags(diags), target(astContext.getTargetInfo()),
95 abi(createCXXABI(cgm&: *this)), genTypes(*this), vtables(*this) {
96
97 // Classic CodeGen's CXX20ModuleInits: C++20 named modules get an
98 // initializer function of their own only under the Itanium mangler, since
99 // no Microsoft mangling for it has been settled on yet; otherwise they fall
100 // back to `_GLOBAL__sub_I_` as in classic CodeGen.
101 cxx20ModuleInits =
102 langOpts.CPlusPlusModules &&
103 getCXXABI().getMangleContext().getKind() == MangleContext::MK_Itanium;
104
105 // Initialize cached types
106 voidTy = cir::VoidType::get(ctx: &getMLIRContext());
107 voidPtrTy = cir::PointerType::get(pointee: voidTy);
108 sInt8Ty = cir::IntType::get(context: &getMLIRContext(), width: 8, /*isSigned=*/is_signed: true);
109 sInt16Ty = cir::IntType::get(context: &getMLIRContext(), width: 16, /*isSigned=*/is_signed: true);
110 sInt32Ty = cir::IntType::get(context: &getMLIRContext(), width: 32, /*isSigned=*/is_signed: true);
111 sInt64Ty = cir::IntType::get(context: &getMLIRContext(), width: 64, /*isSigned=*/is_signed: true);
112 sInt128Ty = cir::IntType::get(context: &getMLIRContext(), width: 128, /*isSigned=*/is_signed: true);
113 uInt8Ty = cir::IntType::get(context: &getMLIRContext(), width: 8, /*isSigned=*/is_signed: false);
114 uInt8PtrTy = cir::PointerType::get(pointee: uInt8Ty);
115 cirAllocaAddressSpace = getTargetCIRGenInfo().getCIRAllocaAddressSpace();
116 uInt16Ty = cir::IntType::get(context: &getMLIRContext(), width: 16, /*isSigned=*/is_signed: false);
117 uInt32Ty = cir::IntType::get(context: &getMLIRContext(), width: 32, /*isSigned=*/is_signed: false);
118 uInt64Ty = cir::IntType::get(context: &getMLIRContext(), width: 64, /*isSigned=*/is_signed: false);
119 uInt128Ty = cir::IntType::get(context: &getMLIRContext(), width: 128, /*isSigned=*/is_signed: false);
120 fP16Ty = cir::FP16Type::get(ctx: &getMLIRContext());
121 bFloat16Ty = cir::BF16Type::get(ctx: &getMLIRContext());
122 floatTy = cir::SingleType::get(ctx: &getMLIRContext());
123 doubleTy = cir::DoubleType::get(ctx: &getMLIRContext());
124 fP80Ty = cir::FP80Type::get(ctx: &getMLIRContext());
125 fP128Ty = cir::FP128Type::get(ctx: &getMLIRContext());
126
127 allocaInt8PtrTy = cir::PointerType::get(pointee: uInt8Ty, addrSpace: cirAllocaAddressSpace);
128
129 PointerAlignInBytes =
130 astContext
131 .toCharUnitsFromBits(
132 BitSize: astContext.getTargetInfo().getPointerAlign(AddrSpace: LangAS::Default))
133 .getQuantity();
134
135 const unsigned charSize = target.getCharWidth();
136 uCharTy = cir::IntType::get(context: &getMLIRContext(), width: charSize, /*isSigned=*/is_signed: false);
137
138 const unsigned sizeTypeSize = target.getTypeWidth(T: target.getSizeType());
139 SizeSizeInBytes = sizeTypeSize / charSize;
140 // In CIRGenTypeCache, UIntPtrTy and SizeType are fields of the same union
141 uIntPtrTy =
142 cir::IntType::get(context: &getMLIRContext(), width: sizeTypeSize, /*isSigned=*/is_signed: false);
143 ptrDiffTy =
144 cir::IntType::get(context: &getMLIRContext(), width: sizeTypeSize, /*isSigned=*/is_signed: true);
145
146 std::optional<cir::SourceLanguage> sourceLanguage = getCIRSourceLanguage();
147 if (sourceLanguage)
148 getModule()->setAttr(
149 name: cir::CIRDialect::getSourceLanguageAttrName(),
150 value: cir::SourceLanguageAttr::get(context: &mlirContext, value: *sourceLanguage));
151 if (langOpts.OpenCL || (langOpts.CUDAIsDevice && getTriple().isSPIRV())) {
152 // CUDA and HIP use OpenCL 2.0 metadata when targeting SPIR-V.
153 unsigned version =
154 langOpts.OpenCL ? langOpts.getOpenCLCompatibleVersion() : 200;
155 setOpenCLVersionAttr(attrName: cir::CIRDialect::getOpenCLVersionAttrName(), version);
156 if (langOpts.OpenCLCPlusPlus)
157 setOpenCLVersionAttr(attrName: cir::CIRDialect::getOpenCLCXXVersionAttrName(),
158 version: langOpts.OpenCLCPlusPlusVersion);
159 // SPIR v2.0 s2.12 requires opencl.spir.version.
160 if (getTriple().isSPIR()) {
161 unsigned spirMajor = version / 100;
162 getModule()->setAttr(name: cir::CIRDialect::getOpenCLSPIRVersionAttrName(),
163 value: cir::OpenCLVersionAttr::get(context: &getMLIRContext(),
164 majorVersion: spirMajor,
165 minorVersion: spirMajor > 1 ? 0 : 2));
166 }
167 }
168 getModule()->setAttr(name: cir::CIRDialect::getTripleAttrName(),
169 value: builder.getStringAttr(bytes: getTriple().str()));
170 getModule()->setAttr(name: cir::CIRDialect::getTargetABIAttrName(),
171 value: builder.getStringAttr(bytes: getTarget().getABI()));
172
173 // Assign the target-cpu and target-features as attributes to the module. This
174 // will allow subsequent passes to use this to ensure that 'generated'
175 // functions get the correct behavior for the model.
176 llvm::StringMap<std::string> defaultCPUAndFeatures;
177 if (getCPUAndFeaturesAttributes(gd: GlobalDecl(), attrs&: defaultCPUAndFeatures))
178 for (const auto &[key, val] : defaultCPUAndFeatures)
179 getModule()->setAttr(name: key, value: builder.getStringAttr(bytes: val));
180
181 // If cxxabi was overridden via -fc++-abi, record it.
182 if (langOpts.CXXABI)
183 getModule()->setAttr(
184 name: cir::CIRDialect::getCXXABIAttrName(),
185 value: builder.getStringAttr(bytes: TargetCXXABI::getSpelling(ABIKind: *langOpts.CXXABI)));
186
187 if (llvm::VersionTuple sdkVersion = getTarget().getSDKVersion();
188 !sdkVersion.empty())
189 getModule()->setAttr(name: cir::CIRDialect::getSDKVersionAttrName(),
190 value: builder.getStringAttr(bytes: sdkVersion.getAsString()));
191 if (langOpts.CUDA && !langOpts.CUID.empty())
192 getModule()->setAttr(name: cir::CIRDialect::getCUIDHashAttrName(),
193 value: builder.getStringAttr(bytes: astContext.getCUIDHash()));
194 // TODO(CIR): These attributes should eventually be replaced by
195 // TypeSizeInfoAttr once it is upstreamed.
196 getModule()->setAttr(name: cir::CIRDialect::getSizeTypeWidthAttrName(),
197 value: builder.getI32IntegerAttr(value: sizeTypeSize));
198 getModule()->setAttr(name: cir::CIRDialect::getIntTypeWidthAttrName(),
199 value: builder.getI32IntegerAttr(value: target.getIntWidth()));
200
201 // Serialize the lowering-relevant LangOptions and CodeGenOptions onto the
202 // ModuleOp so a reloaded .cir is self-describing and lowers the same way it
203 // was compiled, without a live clang::LangOptions or clang::CodeGenOptions.
204 getModule()->setAttr(
205 name: cir::CIRDialect::getLoweringOptionsAttrName(),
206 value: cir::LoweringOptionsAttr::get(
207 context: &mlirContext,
208 /*exceptions=*/langOpts.Exceptions,
209 /*threadsafe_statics=*/langOpts.ThreadsafeStatics,
210 /*cuda=*/langOpts.CUDA,
211 /*cuda_is_device=*/langOpts.CUDAIsDevice,
212 /*hip=*/langOpts.HIP,
213 /*hlsl=*/langOpts.HLSL,
214 /*sycl_is_device=*/langOpts.SYCLIsDevice,
215 /*gpu_rdc=*/langOpts.GPURelocatableDeviceCode,
216 /*openmp=*/langOpts.OpenMP != 0,
217 /*openmp_is_target_device=*/langOpts.OpenMPIsTargetDevice,
218 /*clang_abi_compat=*/
219 static_cast<int32_t>(langOpts.getClangABICompat()),
220 long_double_size: static_cast<int32_t>(langOpts.LongDoubleSize),
221 /*enable_noundef_attrs=*/cgo.EnableNoundefAttrs,
222 /*no_trapping_math=*/langOpts.getDefaultExceptionMode() ==
223 LangOptions::FPE_Ignore,
224 /*ssp_buffer_size=*/cgo.SSPBufferSize,
225 /*relative_cxx_abi_vtables=*/langOpts.RelativeCXXABIVTables));
226
227 if (cgo.OptimizationLevel > 0 || cgo.OptimizeSize > 0)
228 getModule()->setAttr(name: cir::CIRDialect::getOptInfoAttrName(),
229 value: cir::OptInfoAttr::get(context: &mlirContext,
230 level: cgo.OptimizationLevel,
231 size: cgo.OptimizeSize));
232
233 getModule()->setAttr(
234 name: cir::CIRDialect::getDefaultTlsModelAttrName(),
235 value: cir::TLSModelAttr::get(context: &mlirContext, value: getDefaultCIRTLSModel()));
236
237 if (langOpts.OpenMP) {
238 mlir::omp::OffloadModuleOpts ompOpts(
239 langOpts.OpenMPTargetDebug, langOpts.OpenMPTeamSubscription,
240 langOpts.OpenMPThreadSubscription, langOpts.OpenMPNoThreadState,
241 langOpts.OpenMPNoNestedParallelism, langOpts.OpenMPIsTargetDevice,
242 getTriple().isGPU(), langOpts.OpenMPForceUSM, langOpts.OpenMP,
243 langOpts.OMPHostIRFile, langOpts.OMPTargetTriples, langOpts.NoGPULib);
244 mlir::omp::setOffloadModuleInterfaceAttributes(module: getModule(), opts: ompOpts);
245 mlir::omp::setOpenMPVersionAttribute(module: getModule(), version: langOpts.OpenMP);
246 }
247
248 if (langOpts.CUDA)
249 createCUDARuntime();
250 if (langOpts.OpenMP)
251 createOpenMPRuntime();
252
253 // Set the module name to be the name of the main file. TranslationUnitDecl
254 // often contains invalid source locations and isn't a reliable source for the
255 // module location.
256 FileID mainFileId = astContext.getSourceManager().getMainFileID();
257 const FileEntry &mainFile =
258 *astContext.getSourceManager().getFileEntryForID(FID: mainFileId);
259 StringRef path = mainFile.tryGetRealPathName();
260 if (!path.empty()) {
261 getModule().setSymName(path);
262 getModule()->setLoc(mlir::FileLineColLoc::get(context: &mlirContext, fileName: path,
263 /*line=*/0,
264 /*column=*/0));
265 }
266}
267
268CIRGenModule::~CIRGenModule() = default;
269
270void CIRGenModule::setOpenCLVersionAttr(StringRef attrName, unsigned version) {
271 getModule()->setAttr(
272 name: attrName, value: cir::OpenCLVersionAttr::get(context: &getMLIRContext(), majorVersion: version / 100,
273 minorVersion: (version % 100) / 10));
274}
275
276void CIRGenModule::createCUDARuntime() {
277 cudaRuntime.reset(p: createNVCUDARuntime(cgm&: *this));
278}
279
280void CIRGenModule::createOpenMPRuntime() {
281 openMPRuntime = std::make_unique<CIRGenOpenMPRuntime>(args&: *this);
282}
283
284/// FIXME: this could likely be a common helper and not necessarily related
285/// with codegen.
286/// Return the best known alignment for an unknown pointer to a
287/// particular class.
288CharUnits CIRGenModule::getClassPointerAlignment(const CXXRecordDecl *rd) {
289 if (!rd->hasDefinition())
290 return CharUnits::One(); // Hopefully won't be used anywhere.
291
292 auto &layout = astContext.getASTRecordLayout(D: rd);
293
294 // If the class is final, then we know that the pointer points to an
295 // object of that type and can use the full alignment.
296 if (rd->isEffectivelyFinal())
297 return layout.getAlignment();
298
299 // Otherwise, we have to assume it could be a subclass.
300 return layout.getNonVirtualAlignment();
301}
302
303CharUnits CIRGenModule::getNaturalTypeAlignment(QualType t,
304 LValueBaseInfo *baseInfo,
305 bool forPointeeType) {
306 assert(!cir::MissingFeatures::opTBAA());
307
308 // FIXME: This duplicates logic in ASTContext::getTypeAlignIfKnown, but
309 // that doesn't return the information we need to compute baseInfo.
310
311 // Honor alignment typedef attributes even on incomplete types.
312 // We also honor them straight for C++ class types, even as pointees;
313 // there's an expressivity gap here.
314 if (const auto *tt = t->getAs<TypedefType>()) {
315 if (unsigned align = tt->getDecl()->getMaxAlignment()) {
316 if (baseInfo)
317 *baseInfo = LValueBaseInfo(AlignmentSource::AttributedType);
318 return astContext.toCharUnitsFromBits(BitSize: align);
319 }
320 }
321
322 bool alignForArray = t->isArrayType();
323
324 // Analyze the base element type, so we don't get confused by incomplete
325 // array types.
326 t = astContext.getBaseElementType(QT: t);
327
328 if (t->isIncompleteType()) {
329 // We could try to replicate the logic from
330 // ASTContext::getTypeAlignIfKnown, but nothing uses the alignment if the
331 // type is incomplete, so it's impossible to test. We could try to reuse
332 // getTypeAlignIfKnown, but that doesn't return the information we need
333 // to set baseInfo. So just ignore the possibility that the alignment is
334 // greater than one.
335 if (baseInfo)
336 *baseInfo = LValueBaseInfo(AlignmentSource::Type);
337 return CharUnits::One();
338 }
339
340 if (baseInfo)
341 *baseInfo = LValueBaseInfo(AlignmentSource::Type);
342
343 CharUnits alignment;
344 const CXXRecordDecl *rd = nullptr;
345 if (t.getQualifiers().hasUnaligned()) {
346 alignment = CharUnits::One();
347 } else if (forPointeeType && !alignForArray &&
348 (rd = t->getAsCXXRecordDecl())) {
349 alignment = getClassPointerAlignment(rd);
350 } else {
351 alignment = astContext.getTypeAlignInChars(T: t);
352 }
353
354 // Cap to the global maximum type alignment unless the alignment
355 // was somehow explicit on the type.
356 if (unsigned maxAlign = astContext.getLangOpts().MaxTypeAlign) {
357 if (alignment.getQuantity() > maxAlign &&
358 !astContext.isAlignmentRequired(T: t))
359 alignment = CharUnits::fromQuantity(Quantity: maxAlign);
360 }
361 return alignment;
362}
363
364CharUnits
365CIRGenModule::getNaturalPointeeTypeAlignment(QualType t,
366 LValueBaseInfo *baseInfo) {
367 return getNaturalTypeAlignment(t: t->getPointeeType(), baseInfo,
368 /*forPointeeType=*/true);
369}
370
371const TargetCIRGenInfo &CIRGenModule::getTargetCIRGenInfo() {
372 if (theTargetCIRGenInfo)
373 return *theTargetCIRGenInfo;
374
375 const llvm::Triple &triple = getTarget().getTriple();
376 switch (triple.getArch()) {
377 default:
378 assert(!cir::MissingFeatures::targetCIRGenInfoArch());
379
380 // Currently we just fall through to x86_64.
381 [[fallthrough]];
382
383 case llvm::Triple::x86_64: {
384 switch (triple.getOS()) {
385 default:
386 assert(!cir::MissingFeatures::targetCIRGenInfoOS());
387
388 // Currently we just fall through to x86_64.
389 [[fallthrough]];
390
391 case llvm::Triple::Linux:
392 theTargetCIRGenInfo = createX8664TargetCIRGenInfo(cgt&: genTypes);
393 return *theTargetCIRGenInfo;
394 }
395 }
396 case llvm::Triple::aarch64:
397 case llvm::Triple::aarch64_32:
398 case llvm::Triple::aarch64_be: {
399 theTargetCIRGenInfo = createAArch64TargetCIRGenInfo(cgt&: genTypes);
400 return *theTargetCIRGenInfo;
401 }
402 case llvm::Triple::nvptx:
403 case llvm::Triple::nvptx64:
404 theTargetCIRGenInfo = createNVPTXTargetCIRGenInfo(cgt&: genTypes);
405 return *theTargetCIRGenInfo;
406 case llvm::Triple::amdgpu: {
407 theTargetCIRGenInfo = createAMDGPUTargetCIRGenInfo(cgt&: genTypes);
408 return *theTargetCIRGenInfo;
409 }
410 case llvm::Triple::spir:
411 case llvm::Triple::spir64:
412 case llvm::Triple::spirv:
413 case llvm::Triple::spirv32:
414 case llvm::Triple::spirv64:
415 theTargetCIRGenInfo = createCommonSPIRTargetCIRGenInfo(cgt&: genTypes);
416 return *theTargetCIRGenInfo;
417 case llvm::Triple::systemz: {
418 // z/OS XPLINK target info is not implemented yet.
419 // We just default to SystemZ ELF Target for now.
420 if (triple.isOSzOS())
421 assert(!cir::MissingFeatures::targetCIRGenInfoOS());
422 theTargetCIRGenInfo = createSystemZTargetCIRGenInfo(cgt&: genTypes);
423 return *theTargetCIRGenInfo;
424 }
425 }
426}
427
428mlir::Location CIRGenModule::getLoc(SourceLocation cLoc) {
429 assert(cLoc.isValid() && "expected valid source location");
430 const SourceManager &sm = astContext.getSourceManager();
431 PresumedLoc pLoc = sm.getPresumedLoc(Loc: cLoc);
432 StringRef filename = pLoc.getFilename();
433 return mlir::FileLineColLoc::get(filename: builder.getStringAttr(bytes: filename),
434 line: pLoc.getLine(), column: pLoc.getColumn());
435}
436
437mlir::Location CIRGenModule::getLoc(SourceRange cRange) {
438 assert(cRange.isValid() && "expected a valid source range");
439 mlir::Location begin = getLoc(cLoc: cRange.getBegin());
440 mlir::Location end = getLoc(cLoc: cRange.getEnd());
441 mlir::Attribute metadata;
442 return mlir::FusedLoc::get(locs: {begin, end}, metadata, context: builder.getContext());
443}
444
445mlir::Operation *
446CIRGenModule::getAddrOfGlobal(GlobalDecl gd, ForDefinition_t isForDefinition) {
447 const Decl *d = gd.getDecl();
448
449 if (isa<CXXConstructorDecl>(Val: d) || isa<CXXDestructorDecl>(Val: d))
450 return getAddrOfCXXStructor(gd, /*FnInfo=*/fnInfo: nullptr, /*FnType=*/fnType: nullptr,
451 /*DontDefer=*/dontDefer: false, isForDefinition);
452
453 if (isa<CXXMethodDecl>(Val: d)) {
454 const CIRGenFunctionInfo &fi =
455 getTypes().arrangeCXXMethodDeclaration(md: cast<CXXMethodDecl>(Val: d));
456 cir::FuncType ty = getTypes().getFunctionType(info: fi);
457 return getAddrOfFunction(gd, funcType: ty, /*ForVTable=*/forVTable: false, /*DontDefer=*/dontDefer: false,
458 isForDefinition);
459 }
460
461 if (isa<FunctionDecl>(Val: d)) {
462 const CIRGenFunctionInfo &fi = getTypes().arrangeGlobalDeclaration(gd);
463 cir::FuncType ty = getTypes().getFunctionType(info: fi);
464 return getAddrOfFunction(gd, funcType: ty, /*ForVTable=*/forVTable: false, /*DontDefer=*/dontDefer: false,
465 isForDefinition);
466 }
467
468 return getOrCreateCIRGlobal(d: cast<VarDecl>(Val: d), /*ty=*/nullptr,
469 isForDefinition);
470}
471
472void CIRGenModule::emitGlobalDecl(const clang::GlobalDecl &d) {
473 // We call getAddrOfGlobal with isForDefinition set to ForDefinition in
474 // order to get a Value with exactly the type we need, not something that
475 // might have been created for another decl with the same mangled name but
476 // different type.
477 mlir::Operation *op = getAddrOfGlobal(gd: d, isForDefinition: ForDefinition);
478
479 // In case of different address spaces, we may still get a cast, even with
480 // IsForDefinition equal to ForDefinition. Query mangled names table to get
481 // GlobalValue.
482 if (!op)
483 op = getGlobalValue(ref: getMangledName(gd: d));
484
485 assert(op && "expected a valid global op");
486
487 // Check to see if we've already emitted this. This is necessary for a
488 // couple of reasons: first, decls can end up in deferred-decls queue
489 // multiple times, and second, decls can end up with definitions in unusual
490 // ways (e.g. by an extern inline function acquiring a strong function
491 // redefinition). Just ignore those cases.
492 // TODO: Not sure what to map this to for MLIR
493 mlir::Operation *globalValueOp = op;
494 if (auto gv = dyn_cast<cir::GetGlobalOp>(Val: op)) {
495 globalValueOp = getGlobalValue(ref: gv.getName());
496 assert(globalValueOp && "expected a valid global op");
497 }
498
499 if (auto cirGlobalValue =
500 dyn_cast<cir::CIRGlobalValueInterface>(Val: globalValueOp))
501 if (!cirGlobalValue.isDeclaration())
502 return;
503
504 // If this is OpenMP, check if it is legal to emit this global normally.
505 assert(!cir::MissingFeatures::openMP());
506
507 // Otherwise, emit the definition and move on to the next one.
508 emitGlobalDefinition(gd: d, op);
509}
510
511void CIRGenModule::emitDeferred() {
512 // Emit code for any potentially referenced deferred decls. Since a previously
513 // unused static decl may become used during the generation of code for a
514 // static function, iterate until no changes are made.
515
516 assert(!cir::MissingFeatures::openMP());
517
518 emitDeferredVTables();
519 // Emitting a vtable doesn't directly cause more vtables to
520 // become deferred, although it can cause functions to be
521 // emitted that then need those vtables.
522 assert(deferredVTables.empty());
523
524 // Emit CUDA/HIP static device variables referenced by host code only.
525 // Note we should not clear CUDADeviceVarODRUsedByHost since it is still
526 // needed for further handling.
527 if (getLangOpts().CUDA && getLangOpts().CUDAIsDevice)
528 llvm::append_range(C&: deferredDeclsToEmit,
529 R&: getASTContext().CUDADeviceVarODRUsedByHost);
530
531 // Stop if we're out of both deferred vtables and deferred declarations.
532 if (deferredDeclsToEmit.empty())
533 return;
534
535 // Grab the list of decls to emit. If emitGlobalDefinition schedules more
536 // work, it will not interfere with this.
537 std::vector<GlobalDecl> curDeclsToEmit;
538 curDeclsToEmit.swap(x&: deferredDeclsToEmit);
539
540 for (const GlobalDecl &d : curDeclsToEmit) {
541 // Functions declared with the sycl_kernel_entry_point attribute are
542 // emitted normally during host compilation. During device compilation, a
543 // SYCL kernel caller offload entry point function is generated and emitted
544 // in place of each of these functions.
545 if (const auto *fd = d.getDecl()->getAsFunction()) {
546 if (langOpts.SYCLIsDevice && fd->hasAttr<SYCLKernelEntryPointAttr>() &&
547 fd->isDefined()) {
548 // Functions with an invalid sycl_kernel_entry_point attribute are
549 // ignored during device compilation.
550 if (!fd->getAttr<SYCLKernelEntryPointAttr>()->isInvalidAttr()) {
551 // Generate and emit the SYCL kernel caller function.
552 emitSYCLKernelCaller(kernelEntryPointFn: fd, ctx&: getASTContext());
553 // Recurse to emit any symbols directly or indirectly referenced
554 // by the SYCL kernel caller function.
555 emitDeferred();
556 }
557 // Do not emit the sycl_kernel_entry_point attributed function.
558 continue;
559 }
560 }
561
562 emitGlobalDecl(d);
563
564 // If we found out that we need to emit more decls, do that recursively.
565 // This has the advantage that the decls are emitted in a DFS and related
566 // ones are close together, which is convenient for testing.
567 if (!deferredVTables.empty() || !deferredDeclsToEmit.empty()) {
568 emitDeferred();
569 assert(deferredVTables.empty() && deferredDeclsToEmit.empty());
570 }
571 }
572}
573
574void CIRGenModule::printPostfixForExternalizedDecl(llvm::raw_ostream &os,
575 const Decl *d) {
576 // ptxas does not allow '.' in symbol names. On the other hand, HIP prefers
577 // postfix beginning with '.' since the symbol name can be demangled.
578 if (langOpts.HIP)
579 os << (isa<VarDecl>(Val: d) ? ".static." : ".intern.");
580 else
581 os << (isa<VarDecl>(Val: d) ? "__static__" : "__intern__");
582
583 // If the CUID is not specified we try to generate a unique postfix.
584 if (getLangOpts().CUID.empty()) {
585 // TODO: Once we add 'PreprocessorOpts' into CIRGenModule this part can be
586 // brought in from OG.
587 errorNYI(d->getSourceRange(),
588 "printPostfixForExternalizedDecl: CUID is not specified");
589 } else {
590 os << getASTContext().getCUIDHash();
591 }
592}
593
594static bool shouldSkipAliasEmission(const CIRGenModule &cgm,
595 const ValueDecl *global) {
596 const LangOptions &langOpts = cgm.getLangOpts();
597 if (!langOpts.OpenMPIsTargetDevice && !langOpts.CUDA)
598 return false;
599
600 const auto *aa = global->getAttr<AliasAttr>();
601 GlobalDecl aliaseeGD;
602
603 // Check if the aliasee exists, if the aliasee is not found, skip the alias
604 // emission. This is executed for both the host and device.
605 if (!cgm.lookupRepresentativeDecl(mangledName: aa->getAliasee(), gd&: aliaseeGD))
606 return true;
607
608 const auto *aliaseeDecl = dyn_cast<ValueDecl>(Val: aliaseeGD.getDecl());
609 if (langOpts.OpenMPIsTargetDevice)
610 return !aliaseeDecl ||
611 !OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD: aliaseeDecl);
612
613 // CUDA / HIP
614 const bool hasDeviceAttr = global->hasAttr<CUDADeviceAttr>();
615 const bool aliaseeHasDeviceAttr =
616 aliaseeDecl && aliaseeDecl->hasAttr<CUDADeviceAttr>();
617
618 if (langOpts.CUDAIsDevice)
619 return !hasDeviceAttr || !aliaseeHasDeviceAttr;
620
621 // CUDA / HIP Host
622 // we know that the aliasee exists from above, so we know to emit
623 return false;
624}
625
626void CIRGenModule::emitGlobal(clang::GlobalDecl gd) {
627 if (const auto *cd = dyn_cast<clang::OpenACCConstructDecl>(Val: gd.getDecl())) {
628 emitGlobalOpenACCDecl(cd);
629 return;
630 }
631
632 const auto *global = cast<ValueDecl>(Val: gd.getDecl());
633
634 // Weak references don't produce any output by themselves.
635 if (global->hasAttr<WeakRefAttr>())
636 return;
637
638 // If this is an alias definition (which otherwise looks like a declaration)
639 // emit it now.
640 if (global->hasAttr<AliasAttr>()) {
641 if (shouldSkipAliasEmission(cgm: *this, global))
642 return;
643 emitAliasDefinition(gd);
644 return;
645 }
646
647 // If this is CUDA, be selective about which declarations we emit.
648 if (langOpts.CUDA && !getASTContext().shouldEmitCUDADecl(D: global))
649 return;
650
651 if (langOpts.OpenMP) {
652 // If this is OpenMP, check if it is legal to emit this global normally.
653 if (openMPRuntime && openMPRuntime->emitTargetGlobal(gd))
654 return;
655 if (auto *drd = dyn_cast<OMPDeclareReductionDecl>(Val: global)) {
656 if (mustBeEmitted(d: global))
657 emitOMPDeclareReduction(d: drd);
658 return;
659 }
660 if (auto *dmd = dyn_cast<OMPDeclareMapperDecl>(Val: global)) {
661 if (mustBeEmitted(d: global))
662 emitOMPDeclareMapper(d: dmd);
663 return;
664 }
665 }
666
667 if (const auto *fd = dyn_cast<FunctionDecl>(Val: global)) {
668 // Update deferred annotations with the latest declaration if the function
669 // was already used or defined.
670 if (fd->hasAttr<AnnotateAttr>()) {
671 StringRef mangledName = getMangledName(gd);
672 if (getGlobalValue(ref: mangledName))
673 deferredAnnotations[mangledName] = fd;
674 }
675 if (!fd->doesThisDeclarationHaveABody()) {
676 if (!fd->doesDeclarationForceExternallyVisibleDefinition() &&
677 (!fd->isMultiVersion() || !getTarget().getTriple().isAArch64()))
678 return;
679
680 const CIRGenFunctionInfo &fi = getTypes().arrangeGlobalDeclaration(gd);
681 cir::FuncType ty = getTypes().getFunctionType(info: fi);
682 getAddrOfFunction(gd, funcType: ty, /*ForVTable=*/forVTable: false, /*DontDefer=*/dontDefer: false);
683 return;
684 }
685 } else {
686 const auto *vd = cast<VarDecl>(Val: global);
687 assert(vd->isFileVarDecl() && "Cannot emit local var decl as global.");
688 if (vd->isThisDeclarationADefinition() != VarDecl::Definition &&
689 !astContext.isMSStaticDataMemberInlineDefinition(VD: vd)) {
690 assert(!cir::MissingFeatures::openMP());
691 // If this declaration may have caused an inline variable definition to
692 // change linkage, make sure that it's emitted.
693 if (astContext.getInlineVariableDefinitionKind(VD: vd) ==
694 ASTContext::InlineVariableDefinitionKind::Strong)
695 getOrCreateCIRGlobal(d: vd, /*ty=*/{}, isForDefinition: NotForDefinition);
696 // Otherwise, we can ignore this declaration. The variable will be emitted
697 // on its first use.
698 return;
699 }
700 }
701
702 // Defer code generation to first use when possible, e.g. if this is an inline
703 // function. If the global must always be emitted, do it eagerly if possible
704 // to benefit from cache locality. Deferring code generation is necessary to
705 // avoid adding initializers to external declarations.
706 if (mustBeEmitted(d: global) && mayBeEmittedEagerly(d: global)) {
707 // Make sure we don't double-emit this declaration.
708 if (auto existing = dyn_cast_if_present<cir::CIRGlobalValueInterface>(
709 Val: getGlobalValue(ref: getMangledName(gd)));
710 existing && !existing.isDeclaration())
711 return;
712
713 // Emit the definition if it can't be deferred.
714 emitGlobalDefinition(gd);
715 return;
716 }
717
718 // If we're deferring emission of a C++ variable with an initializer, remember
719 // the order in which it appeared on the file.
720 assert(!cir::MissingFeatures::deferredCXXGlobalInit());
721
722 llvm::StringRef mangledName = getMangledName(gd);
723 if (getGlobalValue(ref: mangledName) != nullptr) {
724 // The value has already been used and should therefore be emitted.
725 addDeferredDeclToEmit(GD: gd);
726 } else if (mustBeEmitted(d: global)) {
727 // The value must be emitted, but cannot be emitted eagerly.
728 assert(!mayBeEmittedEagerly(global));
729 addDeferredDeclToEmit(GD: gd);
730 } else {
731 // Otherwise, remember that we saw a deferred decl with this name. The first
732 // use of the mangled name will cause it to move into deferredDeclsToEmit.
733 deferredDecls[mangledName] = gd;
734 }
735}
736
737void CIRGenModule::emitGlobalFunctionDefinition(clang::GlobalDecl gd,
738 mlir::Operation *op) {
739 auto const *funcDecl = cast<FunctionDecl>(Val: gd.getDecl());
740 const CIRGenFunctionInfo &fi = getTypes().arrangeGlobalDeclaration(gd);
741 cir::FuncType funcType = getTypes().getFunctionType(info: fi);
742 cir::FuncOp funcOp = dyn_cast_if_present<cir::FuncOp>(Val: op);
743 if (!funcOp || funcOp.getFunctionType() != funcType) {
744 funcOp = getAddrOfFunction(gd, funcType, /*ForVTable=*/forVTable: false,
745 /*DontDefer=*/dontDefer: true, isForDefinition: ForDefinition);
746 }
747
748 // Already emitted.
749 if (!funcOp.isDeclaration())
750 return;
751
752 setFunctionLinkage(gd, f: funcOp);
753 setGVProperties(op: funcOp, d: funcDecl);
754 assert(!cir::MissingFeatures::opFuncMaybeHandleStaticInExternC());
755 maybeSetTrivialComdat(d: *funcDecl, op: funcOp);
756 assert(!cir::MissingFeatures::setLLVMFunctionFEnvAttributes());
757
758 CIRGenFunction cgf(*this, builder);
759 curCGF = &cgf;
760 {
761 mlir::OpBuilder::InsertionGuard guard(builder);
762 cgf.generateCode(gd, fn: funcOp, funcType);
763 }
764 curCGF = nullptr;
765
766 setNonAliasAttributes(gd, op: funcOp);
767 setCIRFunctionAttributesForDefinition(decl: funcDecl, f: funcOp);
768
769 auto getPriority = [this](const auto *attr) -> int {
770 Expr *e = attr->getPriority();
771 if (e)
772 return e->EvaluateKnownConstInt(Ctx: this->getASTContext()).getExtValue();
773 return attr->DefaultPriority;
774 };
775
776 if (const ConstructorAttr *ca = funcDecl->getAttr<ConstructorAttr>())
777 addGlobalCtor(ctor: funcOp, priority: getPriority(ca));
778 if (const DestructorAttr *da = funcDecl->getAttr<DestructorAttr>())
779 addGlobalDtor(dtor: funcOp, priority: getPriority(da));
780
781 if (funcDecl->getAttr<AnnotateAttr>())
782 deferredAnnotations[getMangledName(gd)] = funcDecl;
783
784 if (getLangOpts().OpenMP && funcDecl->hasAttr<OMPDeclareTargetDeclAttr>())
785 getOpenMPRuntime().emitDeclareTargetFunction(fd: funcDecl, funcOp);
786}
787
788/// Track functions to be called before main() runs.
789void CIRGenModule::addGlobalCtor(cir::FuncOp ctor,
790 std::optional<int> priority) {
791 assert(!cir::MissingFeatures::globalCtorLexOrder());
792 assert(!cir::MissingFeatures::globalCtorAssociatedData());
793
794 // Traditional LLVM codegen directly adds the function to the list of global
795 // ctors. In CIR we just add a global_ctor attribute to the function. The
796 // global list is created in LoweringPrepare.
797 //
798 // FIXME(from traditional LLVM): Type coercion of void()* types.
799 ctor.setGlobalCtorPriority(priority);
800}
801
802/// Add a function to the list that will be called when the module is unloaded.
803void CIRGenModule::addGlobalDtor(cir::FuncOp dtor,
804 std::optional<int> priority) {
805 if (codeGenOpts.RegisterGlobalDtorsWithAtExit &&
806 (!getASTContext().getTargetInfo().getTriple().isOSAIX()))
807 errorNYI(loc: dtor.getLoc(), feature: "registerGlobalDtorsWithAtExit");
808
809 // FIXME(from traditional LLVM): Type coercion of void()* types.
810 dtor.setGlobalDtorPriority(priority);
811}
812
813void CIRGenModule::handleCXXStaticMemberVarInstantiation(VarDecl *vd) {
814 VarDecl::DefinitionKind dk = vd->isThisDeclarationADefinition();
815 if ((dk == VarDecl::Definition && vd->hasAttr<DLLImportAttr>()) ||
816 (langOpts.CUDA && !getASTContext().shouldEmitCUDADecl(D: vd)))
817 return;
818
819 TemplateSpecializationKind tsk = vd->getTemplateSpecializationKind();
820 // If we have a definition, this might be a deferred decl. If the
821 // instantiation is explicit, make sure we emit it at the end.
822 if (vd->getDefinition() && tsk == TSK_ExplicitInstantiationDefinition)
823 getOrCreateCIRGlobal(d: vd, /*ty=*/{}, isForDefinition: NotForDefinition);
824
825 emitTopLevelDecl(decl: vd);
826}
827
828mlir::Operation *CIRGenModule::getGlobalValue(StringRef name) {
829 auto it = symbolLookupCache.find(Key: name);
830 return it != symbolLookupCache.end() ? it->second : nullptr;
831}
832
833cir::GlobalOp
834CIRGenModule::createGlobalOp(mlir::Location loc, StringRef name, mlir::Type t,
835 bool isConstant,
836 mlir::ptr::MemorySpaceAttrInterface addrSpace,
837 mlir::Operation *insertPoint) {
838 cir::GlobalOp g;
839 CIRGenBuilderTy &builder = getBuilder();
840
841 {
842 mlir::OpBuilder::InsertionGuard guard(builder);
843
844 // If an insertion point is provided, we're replacing an existing global,
845 // otherwise, create the new global immediately after the last gloabl we
846 // emitted.
847 if (insertPoint) {
848 builder.setInsertionPoint(insertPoint);
849 } else {
850 // Group global operations together at the top of the module.
851 if (lastGlobalOp)
852 builder.setInsertionPointAfter(lastGlobalOp);
853 else
854 builder.setInsertionPointToStart(getModule().getBody());
855 }
856
857 g = cir::GlobalOp::create(builder, location: loc, sym_name: name, sym_type: t, isConstant, addrSpace);
858 if (!insertPoint)
859 lastGlobalOp = g;
860
861 // Default to private until we can judge based on the initializer,
862 // since MLIR doesn't allow public declarations.
863 mlir::SymbolTable::setSymbolVisibility(
864 symbol: g, vis: mlir::SymbolTable::Visibility::Private);
865 }
866 symbolLookupCache[g.getSymNameAttr()] = g;
867 return g;
868}
869
870void CIRGenModule::setCommonAttributes(GlobalDecl gd, mlir::Operation *gv) {
871 const Decl *d = gd.getDecl();
872 if (isa_and_nonnull<NamedDecl>(Val: d))
873 setGVProperties(op: gv, d: dyn_cast<NamedDecl>(Val: d));
874 assert(!cir::MissingFeatures::defaultVisibility());
875
876 if (auto gvi = mlir::dyn_cast<cir::CIRGlobalValueInterface>(Val: gv)) {
877 if (d && d->hasAttr<UsedAttr>())
878 addUsedOrCompilerUsedGlobal(gv: gvi);
879
880 if (const auto *vd = dyn_cast_if_present<VarDecl>(Val: d);
881 vd && ((codeGenOpts.KeepPersistentStorageVariables &&
882 (vd->getStorageDuration() == SD_Static ||
883 vd->getStorageDuration() == SD_Thread)) ||
884 (codeGenOpts.KeepStaticConsts &&
885 vd->getStorageDuration() == SD_Static &&
886 vd->getType().isConstQualified())))
887 addUsedOrCompilerUsedGlobal(gv: gvi);
888 }
889}
890
891/// Get the feature delta from the default feature map for the given target CPU.
892static std::vector<std::string>
893getFeatureDeltaFromDefault(const CIRGenModule &cgm, llvm::StringRef targetCPU,
894 llvm::StringMap<bool> &featureMap) {
895 llvm::StringMap<bool> defaultFeatureMap;
896 cgm.getTarget().initFeatureMap(
897 Features&: defaultFeatureMap, Diags&: cgm.getASTContext().getDiagnostics(), CPU: targetCPU, FeatureVec: {});
898
899 std::vector<std::string> delta;
900 for (const auto &[k, v] : featureMap) {
901 auto defaultIt = defaultFeatureMap.find(Key: k);
902 if (defaultIt == defaultFeatureMap.end() || defaultIt->getValue() != v)
903 delta.push_back(x: (v ? "+" : "-") + k.str());
904 }
905
906 return delta;
907}
908
909bool CIRGenModule::getCPUAndFeaturesAttributes(
910 GlobalDecl gd, llvm::StringMap<std::string> &attrs,
911 bool setTargetFeatures) {
912 // Add target-cpu and target-features attributes to functions. If
913 // we have a decl for the function and it has a target attribute then
914 // parse that and add it to the feature set.
915 llvm::StringRef targetCPU = getTarget().getTargetOpts().CPU;
916 llvm::StringRef tuneCPU = getTarget().getTargetOpts().TuneCPU;
917 std::vector<std::string> features;
918 // `fd` may be null when emitting attributes for globals that don't have a
919 // FunctionDecl. The AMDGPU branch below handles
920 // the null case via initFeatureMap.
921 const auto *fd = dyn_cast_or_null<FunctionDecl>(Val: gd.getDecl());
922 fd = fd ? fd->getMostRecentDecl() : fd;
923 const auto *td = fd ? fd->getAttr<TargetAttr>() : nullptr;
924 const auto *tv = fd ? fd->getAttr<TargetVersionAttr>() : nullptr;
925 assert((!td || !tv) && "both target_version and target specified");
926 const auto *sd = fd ? fd->getAttr<CPUSpecificAttr>() : nullptr;
927 const auto *tc = fd ? fd->getAttr<TargetClonesAttr>() : nullptr;
928 bool addedAttr = false;
929 if (td || tv || sd || tc) {
930 llvm::StringMap<bool> featureMap;
931 astContext.getFunctionFeatureMap(FeatureMap&: featureMap, GD: gd);
932
933 // Now add the target-cpu and target-features to the function.
934 // While we populated the feature map above, we still need to
935 // get and parse the target/target_clones attribute so we can
936 // get the cpu for the function.
937 llvm::StringRef featureStr = td ? td->getFeaturesStr() : llvm::StringRef();
938 if (tc && (getTriple().isOSAIX() || getTriple().isX86()))
939 featureStr = tc->getFeatureStr(Index: gd.getMultiVersionIndex());
940 if (!featureStr.empty()) {
941 clang::ParsedTargetAttr parsedAttr =
942 getTarget().parseTargetAttr(Str: featureStr);
943 if (!parsedAttr.CPU.empty() &&
944 getTarget().isValidCPUName(Name: parsedAttr.CPU)) {
945 targetCPU = parsedAttr.CPU;
946 tuneCPU = ""; // Clear the tune CPU.
947 }
948 if (!parsedAttr.Tune.empty() &&
949 getTarget().isValidCPUName(Name: parsedAttr.Tune))
950 tuneCPU = parsedAttr.Tune;
951 }
952
953 if (sd) {
954 // Apply the given CPU name as the 'tune-cpu' so that the optimizer can
955 // favor this processor.
956 tuneCPU = sd->getCPUName(Index: gd.getMultiVersionIndex())->getName();
957 }
958
959 // For AMDGPU, only emit delta features (features that differ from the
960 // target CPU's defaults). Other targets might want to follow a similar
961 // pattern.
962 if (getTarget().getTriple().isAMDGPU()) {
963 features = getFeatureDeltaFromDefault(cgm: *this, targetCPU, featureMap);
964 } else {
965 // Produce the canonical string for this set of features.
966 features.reserve(n: features.size() + featureMap.size());
967 for (const auto &entry : featureMap)
968 features.push_back(x: (entry.getValue() ? "+" : "-") +
969 entry.getKey().str());
970 }
971 } else {
972 // Just add the existing target cpu and target features to the function.
973 if (setTargetFeatures && getTarget().getTriple().isAMDGPU()) {
974 llvm::StringMap<bool> featureMap;
975 if (fd)
976 astContext.getFunctionFeatureMap(FeatureMap&: featureMap, GD: gd);
977 else
978 getTarget().initFeatureMap(Features&: featureMap, Diags&: astContext.getDiagnostics(),
979 CPU: targetCPU,
980 FeatureVec: getTarget().getTargetOpts().Features);
981 features = getFeatureDeltaFromDefault(cgm: *this, targetCPU, featureMap);
982 } else {
983 features = getTarget().getTargetOpts().Features;
984 }
985 }
986
987 if (!targetCPU.empty()) {
988 attrs[cir::CIRDialect::getTargetCPUAttrName()] = targetCPU.str();
989 addedAttr = true;
990 }
991 if (!tuneCPU.empty()) {
992 attrs[cir::CIRDialect::getTuneCPUAttrName()] = tuneCPU.str();
993 addedAttr = true;
994 }
995 if (!features.empty() && setTargetFeatures) {
996 llvm::erase_if(C&: features, P: [&](const std::string &f) {
997 assert(!f.empty() && (f[0] == '+' || f[0] == '-') &&
998 "feature string must start with '+' or '-'");
999 return getTarget().isReadOnlyFeature(Feature: f.substr(pos: 1));
1000 });
1001 llvm::sort(C&: features);
1002 attrs[cir::CIRDialect::getTargetFeaturesAttrName()] =
1003 llvm::join(R&: features, Separator: ",");
1004 addedAttr = true;
1005 }
1006 // TODO(cir): add metadata for AArch64 Function Multi Versioning.
1007 assert(!cir::MissingFeatures::opFuncMultiVersioning());
1008 return addedAttr;
1009}
1010
1011void CIRGenModule::setNonAliasAttributes(GlobalDecl gd, mlir::Operation *op) {
1012 setCommonAttributes(gd, gv: op);
1013
1014 const Decl *d = gd.getDecl();
1015 if (d) {
1016 if (auto gvi = mlir::dyn_cast<cir::CIRGlobalValueInterface>(Val: op)) {
1017 if (const auto *sa = d->getAttr<SectionAttr>())
1018 gvi.setSection(builder.getStringAttr(bytes: sa->getName()));
1019 if (d->hasAttr<RetainAttr>())
1020 addUsedGlobal(gv: gvi);
1021
1022 if (auto func = dyn_cast<cir::FuncOp>(Val: op)) {
1023 llvm::StringMap<std::string> attrs;
1024 if (getCPUAndFeaturesAttributes(gd, attrs)) {
1025 // TODO(cir): Classic codegen also removes fmv-features here, which
1026 // CIR does not emit yet.
1027 //
1028 // getCPUAndFeaturesAttributes reads the most recent declaration, so
1029 // its result supersedes anything an earlier one wrote. Clear first:
1030 // setAttr alone would leave a name this call no longer produces.
1031 for (llvm::StringRef name :
1032 {cir::CIRDialect::getTargetCPUAttrName(),
1033 cir::CIRDialect::getTuneCPUAttrName(),
1034 cir::CIRDialect::getTargetFeaturesAttrName()})
1035 func->removeAttr(name);
1036 for (const auto &[key, val] : attrs)
1037 func->setAttr(name: key, value: builder.getStringAttr(bytes: val));
1038 }
1039 }
1040 }
1041 }
1042
1043 assert(!cir::MissingFeatures::opGlobalPragmaClangSection());
1044 getTargetCIRGenInfo().setTargetAttributes(decl: gd.getDecl(), global: op, module&: *this);
1045}
1046
1047std::optional<cir::SourceLanguage> CIRGenModule::getCIRSourceLanguage() const {
1048 using ClangStd = clang::LangStandard;
1049 using CIRLang = cir::SourceLanguage;
1050 auto opts = getLangOpts();
1051
1052 if (opts.OpenCLCPlusPlus)
1053 return CIRLang::OpenCLCXX;
1054 if (opts.OpenCL)
1055 return CIRLang::OpenCLC;
1056 if (opts.CPlusPlus)
1057 return CIRLang::CXX;
1058 if (opts.C99 || opts.C11 || opts.C17 || opts.C23 || opts.C2y ||
1059 opts.LangStd == ClangStd::lang_c89 ||
1060 opts.LangStd == ClangStd::lang_gnu89)
1061 return CIRLang::C;
1062
1063 // TODO(cir): support remaining source languages.
1064 assert(!cir::MissingFeatures::sourceLanguageCases());
1065 errorNYI(feature: "CIR does not yet support the given source language");
1066 return std::nullopt;
1067}
1068
1069LangAS CIRGenModule::getGlobalVarAddressSpace(const VarDecl *d) {
1070 if (langOpts.OpenCL) {
1071 LangAS as = d ? d->getType().getAddressSpace() : LangAS::opencl_global;
1072 assert(as == LangAS::opencl_global || as == LangAS::opencl_global_device ||
1073 as == LangAS::opencl_global_host || as == LangAS::opencl_constant ||
1074 as == LangAS::opencl_local || as >= LangAS::FirstTargetAddressSpace);
1075 return as;
1076 }
1077
1078 if (langOpts.SYCLIsDevice &&
1079 (!d || d->getType().getAddressSpace() == LangAS::Default))
1080 return LangAS::sycl_global;
1081
1082 if (langOpts.CUDA && langOpts.CUDAIsDevice) {
1083 if (d) {
1084 if (d->hasAttr<CUDAConstantAttr>())
1085 return LangAS::cuda_constant;
1086 if (d->hasAttr<CUDASharedAttr>())
1087 return LangAS::cuda_shared;
1088 if (d->hasAttr<CUDADeviceAttr>())
1089 return LangAS::cuda_device;
1090 if (d->getType().isConstQualified())
1091 return LangAS::cuda_constant;
1092 }
1093 return LangAS::cuda_device;
1094 }
1095
1096 if (langOpts.OpenMP && d && d->hasAttr<OMPAllocateDeclAttr>())
1097 errorNYI(d->getSourceRange(), "OpenMP global address space");
1098
1099 return getTargetCIRGenInfo().getGlobalVarAddressSpace(cgm&: *this, d);
1100}
1101
1102static void setLinkageForGV(cir::GlobalOp &gv, const NamedDecl *nd) {
1103 // Set linkage and visibility in case we never see a definition.
1104 LinkageInfo lv = nd->getLinkageAndVisibility();
1105 // Don't set internal linkage on declarations.
1106 // "extern_weak" is overloaded in LLVM; we probably should have
1107 // separate linkage types for this.
1108 if (isExternallyVisible(L: lv.getLinkage()) &&
1109 (nd->hasAttr<WeakAttr>() || nd->isWeakImported()))
1110 gv.setLinkage(cir::GlobalLinkageKind::ExternalWeakLinkage);
1111}
1112
1113static void setLinkageForFunction(CIRGenModule &cgm, cir::FuncOp &func,
1114 const NamedDecl *nd) {
1115 // Mirrors CodeGenModule::setLinkageForGV for function declarations.
1116 LinkageInfo lv = nd->getLinkageAndVisibility();
1117 if (isExternallyVisible(L: lv.getLinkage()) &&
1118 (nd->hasAttr<WeakAttr>() || nd->isWeakImported())) {
1119 auto linkage = cir::GlobalLinkageKind::ExternalWeakLinkage;
1120 func.setLinkage(linkage);
1121 func.setLinkageAttr(
1122 cir::GlobalLinkageKindAttr::get(context: &cgm.getMLIRContext(), value: linkage));
1123 // Declarations must keep 'private' MLIR visibility; only update for defs.
1124 if (!func.isDeclaration())
1125 mlir::SymbolTable::setSymbolVisibility(
1126 symbol: func, vis: cgm.getMLIRVisibilityFromCIRLinkage(GLK: linkage));
1127 }
1128}
1129
1130static llvm::SmallVector<int64_t> indexesOfArrayAttr(mlir::ArrayAttr indexes) {
1131 llvm::SmallVector<int64_t> inds;
1132 for (mlir::Attribute i : indexes) {
1133 auto ind = mlir::cast<mlir::IntegerAttr>(Val&: i);
1134 inds.push_back(Elt: ind.getValue().getSExtValue());
1135 }
1136 return inds;
1137}
1138
1139static bool isViewOnGlobal(cir::GlobalOp glob, cir::GlobalViewAttr view) {
1140 return view.getSymbol().getValue() == glob.getSymName();
1141}
1142
1143static mlir::Attribute createNewGlobalView(CIRGenModule &cgm,
1144 cir::GlobalOp newGlob,
1145 cir::GlobalViewAttr attr,
1146 mlir::Type oldTy) {
1147 // If the attribute does not require indexes or it is not a global view on
1148 // the global we're replacing, keep the original attribute.
1149 if (!attr.getIndices() || !isViewOnGlobal(glob: newGlob, view: attr))
1150 return attr;
1151
1152 llvm::SmallVector<int64_t> oldInds = indexesOfArrayAttr(indexes: attr.getIndices());
1153 llvm::SmallVector<int64_t> newInds;
1154 CIRGenBuilderTy &bld = cgm.getBuilder();
1155 const cir::CIRDataLayout &layout = cgm.getDataLayout();
1156 mlir::Type newTy = newGlob.getSymType();
1157
1158 uint64_t offset =
1159 bld.computeOffsetFromGlobalViewIndices(layout, ty: oldTy, indices: oldInds);
1160 if (!bld.computeGlobalViewIndicesFromFlatOffset(offset, ty: newTy, layout,
1161 indices&: newInds))
1162 return cir::GlobalOffsetAttr::get(type: attr.getType(), symbol: attr.getSymbol(),
1163 offset: static_cast<int64_t>(offset));
1164
1165 cir::PointerType newPtrTy;
1166
1167 if (isa<cir::RecordType>(Val: oldTy))
1168 newPtrTy = cir::PointerType::get(pointee: newTy);
1169 else if (isa<cir::ArrayType>(Val: oldTy))
1170 newPtrTy = cast<cir::PointerType>(Val: attr.getType());
1171
1172 if (newPtrTy)
1173 return bld.getGlobalViewAttr(type: newPtrTy, globalOp: newGlob, indices: newInds,
1174 addressPoint: attr.getAddressPoint());
1175
1176 // This may be unreachable in practice, but keep it as errorNYI while CIR
1177 // is still under development.
1178 cgm.errorNYI(feature: "Unhandled type in createNewGlobalView");
1179 return {};
1180}
1181
1182static mlir::Attribute getNewInitValue(CIRGenModule &cgm, cir::GlobalOp newGlob,
1183 mlir::Type oldTy,
1184 mlir::Attribute oldInit) {
1185 if (auto oldView = mlir::dyn_cast<cir::GlobalViewAttr>(Val&: oldInit))
1186 return createNewGlobalView(cgm, newGlob, attr: oldView, oldTy);
1187
1188 // A byte offset from a symbol doesn't depend on the symbol's type, so it
1189 // remains valid when the global is replaced.
1190 if (mlir::isa<cir::GlobalOffsetAttr>(Val: oldInit))
1191 return oldInit;
1192
1193 auto getNewInitElements =
1194 [&](mlir::ArrayAttr oldElements) -> mlir::ArrayAttr {
1195 llvm::SmallVector<mlir::Attribute> newElements;
1196 for (mlir::Attribute elt : oldElements) {
1197 if (auto view = mlir::dyn_cast<cir::GlobalViewAttr>(Val&: elt))
1198 newElements.push_back(Elt: createNewGlobalView(cgm, newGlob, attr: view, oldTy));
1199 else if (mlir::isa<cir::ConstArrayAttr, cir::ConstRecordAttr>(Val: elt))
1200 newElements.push_back(Elt: getNewInitValue(cgm, newGlob, oldTy, oldInit: elt));
1201 else
1202 newElements.push_back(Elt: elt);
1203 }
1204 return mlir::ArrayAttr::get(context: cgm.getBuilder().getContext(), value: newElements);
1205 };
1206
1207 if (auto oldArray = mlir::dyn_cast<cir::ConstArrayAttr>(Val&: oldInit)) {
1208 // ConstArrayAttr::verify guarantees the elements are either an ArrayAttr or
1209 // a StringAttr. A StringAttr is a string-literal initializer: raw 8-bit
1210 // character bytes with no nested global references, so there is nothing to
1211 // rewrite and it is returned unchanged. The ArrayAttr case recurses to
1212 // rewrite any nested global views.
1213 mlir::Attribute oldElts = oldArray.getElts();
1214 if (mlir::isa<mlir::StringAttr>(Val: oldElts))
1215 return oldInit;
1216 mlir::Attribute newElements =
1217 getNewInitElements(mlir::cast<mlir::ArrayAttr>(Val&: oldElts));
1218 return cgm.getBuilder().getConstArray(
1219 attrs: newElements, arrayTy: mlir::cast<cir::ArrayType>(Val: oldArray.getType()));
1220 }
1221 if (auto oldRecord = mlir::dyn_cast<cir::ConstRecordAttr>(Val&: oldInit)) {
1222 mlir::ArrayAttr newMembers = getNewInitElements(oldRecord.getMembers());
1223 auto recordTy = mlir::cast<cir::RecordType>(Val: oldRecord.getType());
1224 return cgm.getBuilder().getConstRecordOrZeroAttr(arrayAttr: newMembers, recordTy);
1225 }
1226
1227 // This may be unreachable in practice, but keep it as errorNYI while CIR
1228 // is still under development.
1229 cgm.errorNYI(feature: "Unhandled type in getNewInitValue");
1230 return {};
1231}
1232
1233// We want to replace a global value, but because of CIR's typed pointers,
1234// we need to update the existing uses to reflect the new type, not just replace
1235// them directly.
1236void CIRGenModule::replaceGlobal(cir::GlobalOp oldGV, cir::GlobalOp newGV) {
1237 assert(oldGV.getSymName() == newGV.getSymName() && "symbol names must match");
1238
1239 mlir::Type oldTy = oldGV.getSymType();
1240 mlir::Type newTy = newGV.getSymType();
1241
1242 assert(!cir::MissingFeatures::addressSpace());
1243
1244 // If the type didn't change, why are we here?
1245 assert(oldTy != newTy && "expected type change in replaceGlobal");
1246
1247 // Visit all uses and add handling to fix up the types.
1248 std::optional<mlir::SymbolTable::UseRange> oldSymUses =
1249 oldGV.getSymbolUses(from: getModule());
1250 for (mlir::SymbolTable::SymbolUse use : *oldSymUses) {
1251 mlir::Operation *userOp = use.getUser();
1252 assert(
1253 (mlir::isa<cir::GetGlobalOp, cir::GlobalOp, cir::ConstantOp>(userOp)) &&
1254 "Unexpected user for global op");
1255
1256 if (auto getGlobalOp = dyn_cast<cir::GetGlobalOp>(Val: use.getUser())) {
1257 mlir::Value useOpResultValue = getGlobalOp.getAddr();
1258 useOpResultValue.setType(cir::PointerType::get(pointee: newTy));
1259
1260 mlir::OpBuilder::InsertionGuard guard(builder);
1261 builder.setInsertionPointAfter(getGlobalOp);
1262 mlir::Type ptrTy = builder.getPointerTo(ty: oldTy);
1263 mlir::Value cast =
1264 builder.createBitcast(loc: getGlobalOp->getLoc(), src: useOpResultValue, newTy: ptrTy);
1265 useOpResultValue.replaceAllUsesExcept(newValue: cast, exceptedUser: cast.getDefiningOp());
1266 } else if (auto glob = dyn_cast<cir::GlobalOp>(Val: userOp)) {
1267 if (auto init = glob.getInitialValue()) {
1268 mlir::Attribute nw = getNewInitValue(cgm&: *this, newGlob: newGV, oldTy, oldInit: init.value());
1269 glob.setInitialValueAttr(nw);
1270 }
1271 } else if (auto c = dyn_cast<cir::ConstantOp>(Val: userOp)) {
1272 mlir::Attribute init = getNewInitValue(cgm&: *this, newGlob: newGV, oldTy, oldInit: c.getValue());
1273 auto typedAttr = mlir::cast<mlir::TypedAttr>(Val&: init);
1274 mlir::OpBuilder::InsertionGuard guard(builder);
1275 builder.setInsertionPointAfter(c);
1276 auto newUser = cir::ConstantOp::create(builder, location: c.getLoc(), value: typedAttr);
1277 c.replaceAllUsesWith(op: newUser.getOperation());
1278 c.erase();
1279 }
1280 }
1281
1282 // If the old global is being tracked as the most-recently-created global,
1283 // update it so that subsequent globals are not inserted after a (now
1284 // erased) operation, which would leave them detached from the module.
1285 if (lastGlobalOp == oldGV)
1286 lastGlobalOp = newGV;
1287 if (getLangOpts().CUDA)
1288 getCUDARuntime().handleGlobalReplace(oldGV, newGV);
1289 eraseGlobalSymbol(op: oldGV);
1290 oldGV.erase();
1291}
1292
1293/// If the specified mangled name is not in the module,
1294/// create and return an mlir GlobalOp with the specified type (TODO(cir):
1295/// address space).
1296///
1297/// TODO(cir):
1298/// 1. If there is something in the module with the specified name, return
1299/// it potentially bitcasted to the right type.
1300///
1301/// 2. If \p d is non-null, it specifies a decl that correspond to this. This
1302/// is used to set the attributes on the global when it is first created.
1303///
1304/// 3. If \p isForDefinition is true, it is guaranteed that an actual global
1305/// with type \p ty will be returned, not conversion of a variable with the same
1306/// mangled name but some other type.
1307cir::GlobalOp CIRGenModule::getOrCreateCIRGlobal(
1308 StringRef mangledName, mlir::Type ty, LangAS langAS, mlir::Location loc,
1309 const VarDecl *d, ForDefinition_t isForDefinition) {
1310
1311 // Lookup the entry, lazily creating it if necessary.
1312 cir::GlobalOp entry;
1313 if (mlir::Operation *v = getGlobalValue(name: mangledName)) {
1314 if (!isa<cir::GlobalOp>(Val: v)) {
1315 errorNYI(loc, feature: "getOrCreateCIRGlobal: global with non-GlobalOp type");
1316 return nullptr;
1317 }
1318 entry = cast<cir::GlobalOp>(Val: v);
1319 }
1320
1321 if (entry) {
1322 mlir::ptr::MemorySpaceAttrInterface entryCIRAS = entry.getAddrSpaceAttr();
1323
1324 // A weakref's target is only weak until something else names it.
1325 if (weakRefReferences.erase(Ptr: entry) && d && !d->hasAttr<WeakAttr>())
1326 entry.setLinkage(cir::GlobalLinkageKind::ExternalLinkage);
1327
1328 assert(!cir::MissingFeatures::setDLLStorageClass());
1329
1330 if (langOpts.OpenMP && !langOpts.OpenMPSimd && d)
1331 getOpenMPRuntime().registerTargetGlobalVariable(vd: d);
1332
1333 if (entry.getSymType() == ty &&
1334 cir::normalizeDefaultAddressSpace(addrSpace: entryCIRAS) ==
1335 cir::normalizeDefaultAddressSpace(addrSpace: getGlobalCIRAddressSpace(as: langAS)))
1336 return entry;
1337
1338 // If there are two attempts to define the same mangled name, issue an
1339 // error.
1340 //
1341 // TODO(cir): look at mlir::GlobalValue::isDeclaration for all aspects of
1342 // recognizing the global as a declaration, for now only check if
1343 // initializer is present.
1344 if (isForDefinition && !entry.isDeclaration()) {
1345 errorNYI(loc, feature: "getOrCreateCIRGlobal: global with conflicting type");
1346 }
1347
1348 // Address space check removed because it is unnecessary because CIR records
1349 // address space info in types.
1350
1351 // (If global is requested for a definition, we always need to create a new
1352 // global, not just return a bitcast.)
1353 if (!isForDefinition)
1354 return entry;
1355 }
1356
1357 // Calculate constant storage flag before creating the global. This was moved
1358 // from after the global creation to ensure the constant flag is set correctly
1359 // at creation time, matching the logic used in emitCXXGlobalVarDeclInit.
1360 bool isConstant = false;
1361 if (d) {
1362 QualType declType = d->getType();
1363
1364 // Classic codegen doesn't try to exclude ctor or dtor here, but has a FIXME
1365 // to try to do a better job. So this bit of code does slightly more effort
1366 // to get a more accurate answer. We can try to exclude ctor, but only when
1367 // the type is complete, as otherwise we have to check for
1368 // fields(particularly whether they are mutable).
1369 bool excludeCtor = !declType->isIncompleteType();
1370 bool needsDtor =
1371 d->needsDestruction(Ctx: astContext) == QualType::DK_cxx_destructor;
1372
1373 isConstant = declType.isConstantStorage(Ctx: astContext, ExcludeCtor: excludeCtor,
1374 /*ExcludeDtor=*/!needsDtor);
1375 }
1376
1377 mlir::ptr::MemorySpaceAttrInterface declCIRAS =
1378 getGlobalCIRAddressSpace(as: getGlobalVarAddressSpace(d));
1379
1380 // mlir::SymbolTable::Visibility::Public is the default, no need to explicitly
1381 // mark it as such.
1382 cir::GlobalOp gv = createGlobalOp(loc, name: mangledName, t: ty, isConstant, addrSpace: declCIRAS,
1383 /*insertPoint=*/entry.getOperation());
1384
1385 // If we already created a global with the same mangled name (but different
1386 // type) before, remove it from its parent.
1387 if (entry)
1388 replaceGlobal(oldGV: entry, newGV: gv);
1389
1390 // This is the first use or definition of a mangled name. If there is a
1391 // deferred decl with this name, remember that we need to emit it at the end
1392 // of the file.
1393 auto ddi = deferredDecls.find(x: mangledName);
1394 if (ddi != deferredDecls.end()) {
1395 // Move the potentially referenced deferred decl to the DeferredDeclsToEmit
1396 // list, and remove it from DeferredDecls (since we don't need it anymore).
1397 addDeferredDeclToEmit(GD: ddi->second);
1398 deferredDecls.erase(position: ddi);
1399 }
1400
1401 // Handle things which are present even on external declarations.
1402 if (d) {
1403 if (langOpts.OpenMP && !langOpts.OpenMPSimd)
1404 getOpenMPRuntime().registerTargetGlobalVariable(vd: d);
1405
1406 gv.setAlignmentAttr(getSize(size: astContext.getDeclAlign(D: d)));
1407
1408 setLinkageForGV(gv, nd: d);
1409
1410 if (d->getTLSKind())
1411 setTLSMode(op: gv, d: *d);
1412
1413 setGVProperties(op: gv, d);
1414
1415 // If required by the ABI, treat declarations of static data members with
1416 // inline initializers as definitions.
1417 if (astContext.isMSStaticDataMemberInlineDefinition(VD: d))
1418 errorNYI(d->getSourceRange(),
1419 "getOrCreateCIRGlobal: MS static data member inline definition");
1420
1421 // Emit section information for extern variables.
1422 if (d->hasExternalStorage()) {
1423 if (const SectionAttr *sa = d->getAttr<SectionAttr>())
1424 gv.setSectionAttr(builder.getStringAttr(bytes: sa->getName()));
1425 }
1426
1427 // Handle XCore specific ABI requirements.
1428 if (getTriple().getArch() == llvm::Triple::xcore)
1429 errorNYI(d->getSourceRange(),
1430 "getOrCreateCIRGlobal: XCore specific ABI requirements");
1431
1432 // Check if we a have a const declaration with an initializer, we may be
1433 // able to emit it as available_externally to expose it's value to the
1434 // optimizer.
1435 if (getLangOpts().CPlusPlus && gv.isPublic() &&
1436 d->getType().isConstQualified() && gv.isDeclaration() &&
1437 !d->hasDefinition() && d->hasInit() && !d->hasAttr<DLLImportAttr>())
1438 errorNYI(
1439 d->getSourceRange(),
1440 "getOrCreateCIRGlobal: external const declaration with initializer");
1441 }
1442
1443 if (d &&
1444 d->isThisDeclarationADefinition(astContext) == VarDecl::DeclarationOnly) {
1445 getTargetCIRGenInfo().setTargetAttributes(decl: d, global: gv, module&: *this);
1446 // TODO(cir): set target attributes
1447 // External HIP managed variables needed to be recorded for transformation
1448 // in both device and host compilations.
1449 if (getLangOpts().CUDA && d && d->hasAttr<HIPManagedAttr>() &&
1450 d->hasExternalStorage())
1451 getCUDARuntime().handleVarRegistration(vd: d, var: gv);
1452 }
1453
1454 assert(!cir::MissingFeatures::addressSpace());
1455 return gv;
1456}
1457
1458mlir::Operation *CIRGenModule::getWeakRefReference(const ValueDecl *vd) {
1459 const auto *aa = vd->getAttr<AliasAttr>();
1460 assert(aa && "No alias?");
1461
1462 // See if there is already something with the target's name in the module.
1463 if (mlir::Operation *entry = getGlobalValue(name: aa->getAliasee())) {
1464 // Classic codegen is able to just treat whatever value it gets as the
1465 // 'right' type. This needs to introduce SOME sort of conversion it seems.
1466 // For now, reject any cases that don't match.
1467 if (const auto *fd = dyn_cast<FunctionDecl>(Val: vd)) {
1468 auto fn = dyn_cast<cir::FuncOp>(Val: entry);
1469
1470 if (fn && fn.getFunctionType() == convertType(type: fd->getType()))
1471 return fn;
1472 } else if (auto gv = dyn_cast<cir::GlobalOp>(Val: entry)) {
1473 if (gv.getSymType() == getTypes().convertTypeForMem(vd->getType()))
1474 return gv;
1475 }
1476
1477 errorNYI(vd->getSourceRange(),
1478 "weakref target already declared with a different type");
1479 return nullptr;
1480 }
1481
1482 mlir::Operation *aliasee;
1483 if (const auto *fd = dyn_cast<FunctionDecl>(Val: vd)) {
1484 cir::FuncOp fn =
1485 getOrCreateCIRFunction(mangledName: aa->getAliasee(), funcType: convertType(type: fd->getType()),
1486 gd: GlobalDecl(fd), /*forVTable=*/false);
1487 fn.setLinkage(cir::GlobalLinkageKind::ExternalWeakLinkage);
1488 aliasee = fn;
1489 } else {
1490 cir::GlobalOp gv = getOrCreateCIRGlobal(
1491 mangledName: aa->getAliasee(), ty: getTypes().convertTypeForMem(vd->getType()),
1492 langAS: LangAS::Default, loc: getLoc(cRange: vd->getSourceRange()), /*d=*/nullptr,
1493 isForDefinition: NotForDefinition);
1494 gv.setLinkage(cir::GlobalLinkageKind::ExternalWeakLinkage);
1495 aliasee = gv;
1496 }
1497
1498 weakRefReferences.insert(Ptr: aliasee);
1499 return aliasee;
1500}
1501
1502cir::GlobalOp
1503CIRGenModule::getOrCreateCIRGlobal(const VarDecl *d, mlir::Type ty,
1504 ForDefinition_t isForDefinition) {
1505 assert(d->hasGlobalStorage() && "Not a global variable");
1506 QualType astTy = d->getType();
1507 if (!ty)
1508 ty = getTypes().convertTypeForMem(astTy);
1509
1510 StringRef mangledName = getMangledName(gd: d);
1511 return getOrCreateCIRGlobal(mangledName, ty, langAS: getGlobalVarAddressSpace(d),
1512 loc: getLoc(cRange: d->getSourceRange()), d, isForDefinition);
1513}
1514
1515/// Return the mlir::Value for the address of the given global variable. If
1516/// \p ty is non-null and if the global doesn't exist, then it will be created
1517/// with the specified type instead of whatever the normal requested type would
1518/// be. If \p isForDefinition is true, it is guaranteed that an actual global
1519/// with type \p ty will be returned, not conversion of a variable with the same
1520/// mangled name but some other type.
1521mlir::Value CIRGenModule::getAddrOfGlobalVar(const VarDecl *d, mlir::Type ty,
1522 ForDefinition_t isForDefinition) {
1523 assert(d->hasGlobalStorage() && "Not a global variable");
1524 QualType astTy = d->getType();
1525 if (!ty)
1526 ty = getTypes().convertTypeForMem(astTy);
1527
1528 bool tlsAccess = d->getTLSKind() != VarDecl::TLS_None;
1529 cir::GlobalOp g = getOrCreateCIRGlobal(d, ty, isForDefinition);
1530 mlir::Type ptrTy = builder.getPointerTo(ty: g.getSymType(), as: g.getAddrSpaceAttr());
1531 mlir::Value addr = cir::GetGlobalOp::create(
1532 builder, location: getLoc(cRange: d->getSourceRange()), addr: ptrTy, name: g.getSymNameAttr(),
1533 tls: tlsAccess,
1534 /*static_local=*/g.getDynamicInitGuard().has_value() &&
1535 d->isLocalVarDecl());
1536 return castGlobalToDeclAddrSpace(addr, vd: *d);
1537}
1538
1539mlir::Value CIRGenModule::castGlobalToDeclAddrSpace(mlir::Value addr,
1540 const VarDecl &vd) {
1541 // A global may live in a different address space than its declared type,
1542 // e.g. a CUDA __shared__ variable. Like classic CodeGen, cast once where
1543 // the address is formed so every user sees the declared type.
1544 auto ptrTy = mlir::cast<cir::PointerType>(Val: addr.getType());
1545 mlir::ptr::MemorySpaceAttrInterface declAS =
1546 getTypes().getPointerAddressSpace(pointeeTy: vd.getType());
1547 if (ptrTy.getAddrSpace() == declAS)
1548 return addr;
1549 return builder.createAddrSpaceCast(
1550 src: addr, newTy: builder.getPointerTo(ty: ptrTy.getPointee(), as: declAS));
1551}
1552
1553cir::GlobalViewAttr CIRGenModule::getAddrOfGlobalVarAttr(const VarDecl *d) {
1554 assert(d->hasGlobalStorage() && "Not a global variable");
1555 mlir::Type ty = getTypes().convertTypeForMem(d->getType());
1556
1557 cir::GlobalOp globalOp = getOrCreateCIRGlobal(d, ty, isForDefinition: NotForDefinition);
1558 cir::PointerType ptrTy =
1559 builder.getPointerTo(ty: globalOp.getSymType(), as: globalOp.getAddrSpaceAttr());
1560 return builder.getGlobalViewAttr(type: ptrTy, globalOp);
1561}
1562
1563void CIRGenModule::addUsedGlobal(cir::CIRGlobalValueInterface gv) {
1564 assert((mlir::isa<cir::FuncOp>(gv.getOperation()) ||
1565 !gv.isDeclarationForLinker()) &&
1566 "Only globals with definition can force usage.");
1567 llvmUsed.emplace_back(args&: gv);
1568}
1569
1570void CIRGenModule::addCompilerUsedGlobal(cir::CIRGlobalValueInterface gv) {
1571 assert(!gv.isDeclarationForLinker() &&
1572 "Only globals with definition can force usage.");
1573 llvmCompilerUsed.emplace_back(args&: gv);
1574}
1575
1576void CIRGenModule::addUsedOrCompilerUsedGlobal(
1577 cir::CIRGlobalValueInterface gv) {
1578 assert((mlir::isa<cir::FuncOp>(gv.getOperation()) ||
1579 !gv.isDeclarationForLinker()) &&
1580 "Only globals with definition can force usage.");
1581 if (getTriple().isOSBinFormatELF())
1582 llvmCompilerUsed.emplace_back(args&: gv);
1583 else
1584 llvmUsed.emplace_back(args&: gv);
1585}
1586
1587static void emitUsed(CIRGenModule &cgm, StringRef name,
1588 std::vector<cir::CIRGlobalValueInterface> &list) {
1589 if (list.empty())
1590 return;
1591
1592 CIRGenBuilderTy &builder = cgm.getBuilder();
1593 mlir::Location loc = builder.getUnknownLoc();
1594 cir::PointerType voidPtrTy = builder.getVoidPtrTy(
1595 as: cgm.getTypes().getPointerAddressSpace(pointeeTy: cgm.getASTContext().VoidTy));
1596
1597 llvm::SmallVector<mlir::Attribute> usedArray;
1598 usedArray.resize(N: list.size());
1599 for (auto [i, op] : llvm::enumerate(First&: list)) {
1600 usedArray[i] = cir::GlobalViewAttr::get(
1601 type: voidPtrTy, symbol: mlir::FlatSymbolRefAttr::get(value: op.getNameAttr()));
1602 }
1603
1604 cir::ArrayType arrayTy = cir::ArrayType::get(elementType: voidPtrTy, size: usedArray.size());
1605
1606 cir::ConstArrayAttr initAttr = cir::ConstArrayAttr::get(
1607 type: arrayTy, elts: mlir::ArrayAttr::get(context: &cgm.getMLIRContext(), value: usedArray));
1608
1609 cir::GlobalOp gv = cgm.createGlobalOp(
1610 loc, name, t: arrayTy, /*isConstant=*/false,
1611 addrSpace: cgm.getDataLayout().getDefaultGlobalsAddrSpace(ctx: &cgm.getMLIRContext()));
1612 gv.setLinkage(cir::GlobalLinkageKind::AppendingLinkage);
1613 gv.setInitialValueAttr(initAttr);
1614 gv.setSectionAttr(builder.getStringAttr(bytes: "llvm.metadata"));
1615}
1616
1617void CIRGenModule::emitLLVMUsed() {
1618 emitUsed(cgm&: *this, name: "llvm.used", list&: llvmUsed);
1619 emitUsed(cgm&: *this, name: "llvm.compiler.used", list&: llvmCompilerUsed);
1620}
1621
1622void CIRGenModule::emitGlobalVarDefinition(const clang::VarDecl *vd,
1623 bool isTentative) {
1624 // OpenCL global variables of sampler type are translated to function calls,
1625 // therefore no need to be translated.
1626 if (getLangOpts().OpenCL && vd->getType()->isSamplerT())
1627 return;
1628
1629 if (getLangOpts().OpenMPIsTargetDevice) {
1630 errorNYI(vd->getSourceRange(),
1631 "emitGlobalVarDefinition: emit OpenMP global variable");
1632 return;
1633 }
1634
1635 // Whether the definition of the variable is available externally.
1636 // If yes, we shouldn't emit the GloablCtor and GlobalDtor for the variable
1637 // since this is the job for its original source.
1638 bool isDefinitionAvailableExternally =
1639 astContext.GetGVALinkageForVariable(VD: vd) == GVA_AvailableExternally;
1640
1641 // It is useless to emit the definition for an available_externally variable
1642 // which can't be marked as const.
1643 if (isDefinitionAvailableExternally &&
1644 (!vd->hasConstantInitialization() ||
1645 // TODO: Update this when we have interface to check constexpr
1646 // destructor.
1647 vd->needsDestruction(Ctx: astContext) ||
1648 !vd->getType().isConstantStorage(Ctx: astContext, ExcludeCtor: true, ExcludeDtor: true)))
1649 return;
1650
1651 mlir::Attribute init;
1652 bool needsGlobalCtor = false;
1653 bool needsGlobalDtor =
1654 !isDefinitionAvailableExternally &&
1655 vd->needsDestruction(Ctx: astContext) == QualType::DK_cxx_destructor;
1656 const VarDecl *initDecl;
1657 const Expr *initExpr = vd->getAnyInitializer(D&: initDecl);
1658
1659 std::optional<ConstantEmitter> emitter;
1660
1661 // CUDA E.2.4.1 "__shared__ variables cannot have an initialization
1662 // as part of their declaration." Sema has already checked for
1663 // error cases, so we just need to set Init to PoisonValue.
1664 bool isCUDASharedVar =
1665 getLangOpts().CUDAIsDevice && vd->hasAttr<CUDASharedAttr>();
1666 // Shadows of initialized device-side global variables are also left
1667 // undefined.
1668 // Managed Variables should be initialized on both host side and device side.
1669 bool isCUDAShadowVar =
1670 !getLangOpts().CUDAIsDevice && !vd->hasAttr<HIPManagedAttr>() &&
1671 (vd->hasAttr<CUDAConstantAttr>() || vd->hasAttr<CUDADeviceAttr>() ||
1672 vd->hasAttr<CUDASharedAttr>());
1673 bool isCUDADeviceShadowVar =
1674 getLangOpts().CUDAIsDevice && !vd->hasAttr<HIPManagedAttr>() &&
1675 (vd->getType()->isCUDADeviceBuiltinSurfaceType() ||
1676 vd->getType()->isCUDADeviceBuiltinTextureType());
1677
1678 if (getLangOpts().CUDA &&
1679 (isCUDASharedVar || isCUDAShadowVar || isCUDADeviceShadowVar)) {
1680 init = cir::UndefAttr::get(type: convertType(type: vd->getType()));
1681 } else if (vd->hasAttr<LoaderUninitializedAttr>()) {
1682 errorNYI(vd->getSourceRange(),
1683 "emitGlobalVarDefinition: loader uninitialized attribute");
1684 } else if (!initExpr) {
1685 // This is a tentative definition; tentative definitions are
1686 // implicitly initialized with { 0 }.
1687 //
1688 // Note that tentative definitions are only emitted at the end of
1689 // a translation unit, so they should never have incomplete
1690 // type. In addition, EmitTentativeDefinition makes sure that we
1691 // never attempt to emit a tentative definition if a real one
1692 // exists. A use may still exists, however, so we still may need
1693 // to do a RAUW.
1694 assert(!vd->getType()->isIncompleteType() && "Unexpected incomplete type");
1695 init = emitNullConstantAttr(t: vd->getType());
1696 } else {
1697 emitter.emplace(args&: *this);
1698 mlir::Attribute initializer = emitter->tryEmitForInitializer(d: *initDecl);
1699 if (!initializer) {
1700 QualType qt = initExpr->getType();
1701 if (vd->getType()->isReferenceType())
1702 qt = vd->getType();
1703
1704 if (getLangOpts().CPlusPlus) {
1705 if (initDecl->hasFlexibleArrayInit(Ctx: astContext))
1706 errorNYI(vd->getSourceRange(),
1707 "emitGlobalVarDefinition: flexible array initializer");
1708 init = emitNullConstantAttr(t: qt);
1709 if (!isDefinitionAvailableExternally)
1710 needsGlobalCtor = true;
1711 } else {
1712 errorNYI(vd->getSourceRange(),
1713 "emitGlobalVarDefinition: static initializer");
1714 }
1715 } else {
1716 init = initializer;
1717 // We don't need an initializer, so remove the entry for the delayed
1718 // initializer position (just in case this entry was delayed) if we
1719 // also don't need to register a destructor.
1720 assert(!cir::MissingFeatures::deferredCXXGlobalInit());
1721 }
1722 }
1723
1724 mlir::Type initType;
1725 if (mlir::isa<mlir::SymbolRefAttr>(Val: init)) {
1726 errorNYI(
1727 vd->getSourceRange(),
1728 "emitGlobalVarDefinition: global initializer is a symbol reference");
1729 return;
1730 } else {
1731 assert(mlir::isa<mlir::TypedAttr>(init) && "This should have a type");
1732 auto typedInitAttr = mlir::cast<mlir::TypedAttr>(Val&: init);
1733 initType = typedInitAttr.getType();
1734 }
1735 assert(!mlir::isa<mlir::NoneType>(initType) && "Should have a type by now");
1736
1737 cir::GlobalOp gv =
1738 getOrCreateCIRGlobal(d: vd, ty: initType, isForDefinition: ForDefinition_t(!isTentative));
1739 // TODO(cir): Strip off pointer casts from Entry if we get them?
1740
1741 if (!gv || gv.getSymType() != initType) {
1742 errorNYI(vd->getSourceRange(),
1743 "emitGlobalVarDefinition: global initializer with type mismatch");
1744 return;
1745 }
1746
1747 assert(!cir::MissingFeatures::maybeHandleStaticInExternC());
1748
1749 if (vd->hasAttr<AnnotateAttr>())
1750 addGlobalAnnotations(d: vd, gv);
1751
1752 // Set CIR's linkage type as appropriate.
1753 cir::GlobalLinkageKind linkage = getCIRLinkageVarDefinition(vd);
1754
1755 // CUDA B.2.1 "The __device__ qualifier declares a variable that resides on
1756 // the device. [...]"
1757 // CUDA B.2.2 "The __constant__ qualifier, optionally used together with
1758 // __device__, declares a variable that: [...]
1759 // Is accessible from all the threads within the grid and from the host
1760 // through the runtime library (cudaGetSymbolAddress() / cudaGetSymbolSize()
1761 // / cudaMemcpyToSymbol() / cudaMemcpyFromSymbol())."
1762 if (langOpts.CUDA) {
1763 if (langOpts.CUDAIsDevice) {
1764 // __shared__ variables is not marked as externally initialized,
1765 // because they must not be initialized.
1766 if (linkage != cir::GlobalLinkageKind::InternalLinkage &&
1767 !vd->isConstexpr() && !vd->getType().isConstQualified() &&
1768 (vd->hasAttr<CUDADeviceAttr>() || vd->hasAttr<CUDAConstantAttr>() ||
1769 vd->getType()->isCUDADeviceBuiltinSurfaceType() ||
1770 vd->getType()->isCUDADeviceBuiltinTextureType())) {
1771 gv->setAttr(name: cir::CUDAExternallyInitializedAttr::getMnemonic(),
1772 value: cir::CUDAExternallyInitializedAttr::get(ctx: &getMLIRContext()));
1773 }
1774 } else {
1775 // Adjust linkage of shadow variables in host compilation
1776 getCUDARuntime().internalizeDeviceSideVar(d: vd, linkage);
1777 }
1778 getCUDARuntime().handleVarRegistration(vd, var: gv);
1779 }
1780
1781 // Set initializer and finalize emission
1782 CIRGenModule::setInitializer(op&: gv, value: init);
1783 if (emitter)
1784 emitter->finalize(gv);
1785
1786 // If it is safe to mark the global 'constant', do so now.
1787 // Use the same logic as classic codegen EmitGlobalVarDefinition.
1788 gv.setConstant((vd->hasAttr<CUDAConstantAttr>() && langOpts.CUDAIsDevice) ||
1789 (!needsGlobalCtor && !needsGlobalDtor &&
1790 vd->getType().isConstantStorage(Ctx: astContext,
1791 /*ExcludeCtor=*/true,
1792 /*ExcludeDtor=*/true)));
1793 // If it is in a read-only section, mark it 'constant'.
1794 if (const SectionAttr *sa = vd->getAttr<SectionAttr>()) {
1795 const ASTContext::SectionInfo &si = astContext.SectionInfos[sa->getName()];
1796 if ((si.SectionFlags & ASTContext::PSF_Write) == 0)
1797 gv.setConstant(true);
1798 }
1799
1800 // The global may have been created without a declaration, e.g. as the target
1801 // of a weakref, so make sure the alignment comes from the definition.
1802 gv.setAlignmentAttr(getSize(size: astContext.getDeclAlign(D: vd)));
1803
1804 // Set CIR linkage and DLL storage class.
1805 gv.setLinkage(linkage);
1806 // FIXME(cir): setLinkage should likely set MLIR's visibility automatically.
1807 gv.setVisibility(getMLIRVisibilityFromCIRLinkage(GLK: linkage));
1808 assert(!cir::MissingFeatures::opGlobalDLLImportExport());
1809 if (linkage == cir::GlobalLinkageKind::CommonLinkage) {
1810 // common vars aren't constant even if declared const.
1811 gv.setConstant(false);
1812 // Tentative definition of global variables may be initialized with
1813 // non-zero null pointers. In this case they should have weak linkage
1814 // since common linkage must have zero initializer and must not have
1815 // explicit section therefore cannot have non-zero initial value.
1816 std::optional<mlir::Attribute> initializer = gv.getInitialValue();
1817 if (initializer && !getBuilder().isNullValue(attr: *initializer))
1818 gv.setLinkage(cir::GlobalLinkageKind::WeakAnyLinkage);
1819 }
1820
1821 setNonAliasAttributes(gd: vd, op: gv);
1822
1823 if (vd->getTLSKind() && !vd->isStaticLocal())
1824 setTLSMode(op: gv, d: *vd);
1825
1826 maybeSetTrivialComdat(d: *vd, op: gv);
1827
1828 // Emit the initializer function if necessary.
1829 if (needsGlobalCtor || needsGlobalDtor) {
1830 // We need to make sure that these are emitted 'in order' of definition,
1831 // rather than reference, so make sure this is now 'last', if it needs a
1832 // global ctor/dtor, so that initialization happens in the correct order.
1833 // Other uses of 'lastGlobalOp' just make sure we emit all our globals
1834 // before functions/etc, so it might have ended up out of order/referenced
1835 // earlier, so this is a 'move to the end' once we see definition thing
1836 // happening here.
1837 if (lastGlobalOp != gv.getOperation()) {
1838 if (lastGlobalOp)
1839 gv->moveAfter(existingOp: lastGlobalOp);
1840 else
1841 gv->moveBefore(block: getModule().getBody(), iterator: getModule().getBody()->begin());
1842
1843 lastGlobalOp = gv.getOperation();
1844 }
1845
1846 emitCXXGlobalVarDeclInitFunc(vd, addr: gv, performInit: needsGlobalCtor);
1847 }
1848}
1849
1850bool CIRGenModule::shouldEmitFunction(GlobalDecl gd) {
1851 if (getFunctionLinkage(gd) !=
1852 cir::GlobalLinkageKind::AvailableExternallyLinkage)
1853 return true;
1854
1855 const auto *fd = cast<FunctionDecl>(Val: gd.getDecl());
1856 // Inline builtins must be emitted; the body is redirected to a `.inline`
1857 // symbol in CIRGenFunction::generateCode.
1858 if (fd->isInlineBuiltinDeclaration())
1859 return true;
1860
1861 if (codeGenOpts.OptimizationLevel == 0 && !fd->hasAttr<AlwaysInlineAttr>())
1862 return false;
1863
1864 // We don't import function bodies from other named module units since that
1865 // behavior may break ABI compatibility of the current unit.
1866 if (const Module *m = fd->getOwningModule();
1867 m && m->getTopLevelModule()->isNamedModule() &&
1868 getASTContext().getCurrentNamedModule() != m->getTopLevelModule()) {
1869 // There are practices to mark template member function as always-inline
1870 // and mark the template as extern explicit instantiation but not give
1871 // the definition for member function. So we have to emit the function
1872 // from explicitly instantiation with always-inline.
1873 //
1874 // See https://github.com/llvm/llvm-project/issues/86893 for details.
1875 //
1876 // TODO: Maybe it is better to give it a warning if we call a non-inline
1877 // function from other module units which is marked as always-inline.
1878 if (!fd->isTemplateInstantiation() || !fd->hasAttr<AlwaysInlineAttr>()) {
1879 return false;
1880 }
1881 }
1882
1883 if (fd->hasAttr<NoInlineAttr>())
1884 return false;
1885
1886 // PR9614 / glibc btowc workaround: an available_externally function whose
1887 // body just calls itself (via asm label or __builtin_* lowering on the
1888 // same name) is not a valid stand-in for the real implementation. Drop
1889 // it from the IR so the optimizer doesn't reason about its body.
1890 return !getCXXABI().getMangleContext().isTriviallyRecursive(FD: fd);
1891}
1892
1893void CIRGenModule::emitGlobalDefinition(clang::GlobalDecl gd,
1894 mlir::Operation *op) {
1895 const auto *decl = cast<ValueDecl>(Val: gd.getDecl());
1896 if (const auto *fd = dyn_cast<FunctionDecl>(Val: decl)) {
1897 if (!shouldEmitFunction(gd))
1898 return;
1899
1900 if (const auto *method = dyn_cast<CXXMethodDecl>(Val: decl)) {
1901 // Make sure to emit the definition(s) before we emit the thunks. This is
1902 // necessary for the generation of certain thunks.
1903 if (isa<CXXConstructorDecl>(Val: method) || isa<CXXDestructorDecl>(Val: method))
1904 abi->emitCXXStructor(gd);
1905 else if (fd->isMultiVersion())
1906 errorNYI(method->getSourceRange(), "multiversion functions");
1907 else
1908 emitGlobalFunctionDefinition(gd, op);
1909
1910 if (method->isVirtual())
1911 getVTables().emitThunks(gd);
1912
1913 return;
1914 }
1915
1916 if (fd->isMultiVersion())
1917 errorNYI(fd->getSourceRange(), "multiversion functions");
1918 emitGlobalFunctionDefinition(gd, op);
1919 return;
1920 }
1921
1922 if (const auto *vd = dyn_cast<VarDecl>(Val: decl))
1923 return emitGlobalVarDefinition(vd, isTentative: !vd->hasDefinition());
1924
1925 llvm_unreachable("Invalid argument to CIRGenModule::emitGlobalDefinition");
1926}
1927
1928mlir::Attribute
1929CIRGenModule::getConstantArrayFromStringLiteral(const StringLiteral *e) {
1930 assert(!e->getType()->isPointerType() && "Strings are always arrays");
1931
1932 // Don't emit it as the address of the string, emit the string data itself
1933 // as an inline array.
1934 if (e->getCharByteWidth() == 1) {
1935 SmallString<64> str(e->getString());
1936
1937 // Resize the string to the right size, which is indicated by its type.
1938 const ConstantArrayType *cat =
1939 astContext.getAsConstantArrayType(T: e->getType());
1940 uint64_t finalSize = cat->getZExtSize();
1941 str.resize(N: finalSize);
1942
1943 mlir::Type eltTy = convertType(type: cat->getElementType());
1944 return builder.getString(str, eltTy, size: finalSize, /*ensureNullTerm=*/false);
1945 }
1946
1947 auto arrayTy = mlir::cast<cir::ArrayType>(Val: convertType(type: e->getType()));
1948
1949 auto arrayEltTy = mlir::cast<cir::IntType>(Val: arrayTy.getElementType());
1950
1951 uint64_t arraySize = arrayTy.getSize();
1952 unsigned literalSize = e->getLength();
1953 assert(arraySize > literalSize &&
1954 "wide string literal array size must have room for null terminator?");
1955
1956 // Check if the string is all null bytes before building the vector.
1957 // In most non-zero cases, this will break out on the first element.
1958 bool isAllZero = true;
1959 for (unsigned i = 0; i < literalSize; ++i) {
1960 if (e->getCodeUnit(I: i) != 0) {
1961 isAllZero = false;
1962 break;
1963 }
1964 }
1965
1966 if (isAllZero)
1967 return cir::ZeroAttr::get(type: arrayTy);
1968
1969 // Otherwise emit a constant array holding the characters.
1970 SmallVector<mlir::Attribute> elements;
1971 elements.reserve(N: arraySize);
1972 for (unsigned i = 0; i < literalSize; ++i)
1973 elements.push_back(Elt: cir::IntAttr::get(
1974 type: arrayEltTy, value: llvm::APInt(arrayEltTy.getWidth(), e->getCodeUnit(I: i))));
1975
1976 auto elementsAttr = mlir::ArrayAttr::get(context: &getMLIRContext(), value: elements);
1977 return builder.getConstArray(attrs: elementsAttr, arrayTy);
1978}
1979
1980bool CIRGenModule::supportsCOMDAT() const {
1981 return getTriple().supportsCOMDAT();
1982}
1983
1984void CIRGenModule::maybeSetTrivialComdat(const Decl &d, mlir::Operation *op) {
1985 if (!CodeGenUtils::shouldBeInCOMDAT(Ctx: getASTContext(), D: d))
1986 return;
1987 if (auto globalOp = dyn_cast_or_null<cir::GlobalOp>(Val: op)) {
1988 globalOp.setSelfComdat();
1989 } else {
1990 auto funcOp = cast<cir::FuncOp>(Val: op);
1991 funcOp.setSelfComdat();
1992 }
1993}
1994
1995void CIRGenModule::updateCompletedType(const TagDecl *td) {
1996 // Make sure that this type is translated.
1997 genTypes.updateCompletedType(td);
1998}
1999
2000void CIRGenModule::addReplacement(StringRef name, mlir::Operation *op) {
2001 replacements[name] = op;
2002}
2003
2004#ifndef NDEBUG
2005static bool verifyPointerTypeArgs(llvm::ArrayRef<mlir::Operation *> users,
2006 cir::FuncOp newF) {
2007 for (mlir::Operation *user : users) {
2008 auto call = mlir::dyn_cast<cir::CallOp>(user);
2009 if (!call)
2010 continue;
2011
2012 for (auto [argOp, fnArgType] :
2013 llvm::zip(call.getArgs(), newF.getFunctionType().getInputs())) {
2014 if (argOp.getType() != fnArgType)
2015 return false;
2016 }
2017 }
2018
2019 return true;
2020}
2021#endif // NDEBUG
2022
2023void CIRGenModule::applyReplacements() {
2024 if (replacements.empty())
2025 return;
2026
2027 // Collect the users of the functions being replaced in one walk. Skip CIR
2028 // types. None can hold a symbol reference (see CIRTypes.td), and walking
2029 // them descends into every nested struct of every signature.
2030 llvm::DenseMap<mlir::StringAttr, llvm::SetVector<mlir::Operation *>> users;
2031 for (auto &i : replacements)
2032 users.try_emplace(Key: mlir::StringAttr::get(context: &getMLIRContext(), bytes: i.first));
2033
2034 mlir::ModuleOp module = getModule();
2035 module->walk<mlir::WalkOrder::PreOrder>(callback: [&](mlir::Operation *op) {
2036 auto collect = [&](mlir::Attribute attr) {
2037 attr.walk<mlir::WalkOrder::PreOrder>(
2038 walkFns: [&](mlir::SymbolRefAttr ref) {
2039 auto it = users.find(Val: ref.getRootReference());
2040 if (it != users.end())
2041 it->second.insert(X: op);
2042 return mlir::WalkResult::skip();
2043 },
2044 walkFns: [&](mlir::Type type) {
2045 return mlir::isa<cir::CIRDialect>(Val: type.getDialect())
2046 ? mlir::WalkResult::skip()
2047 : mlir::WalkResult::advance();
2048 });
2049 };
2050 collect(op->getRawDictionaryAttrs());
2051 op->getName().walkInherentAttrs(
2052 op, visitor: [&](llvm::StringRef, mlir::Attribute &attr) { collect(attr); });
2053 // Like mlir::SymbolUserMap, don't look into nested symbol tables.
2054 if (op != module.getOperation() &&
2055 op->hasTrait<mlir::OpTrait::SymbolTable>())
2056 return mlir::WalkResult::skip();
2057 return mlir::WalkResult::advance();
2058 });
2059
2060 for (auto &i : replacements) {
2061 StringRef mangledName = i.first;
2062 mlir::Operation *replacement = i.second;
2063 mlir::Operation *entry = getGlobalValue(name: mangledName);
2064 if (!entry)
2065 continue;
2066 assert(isa<cir::FuncOp>(entry) && "expected function");
2067 auto oldF = cast<cir::FuncOp>(Val: entry);
2068 auto newF = dyn_cast<cir::FuncOp>(Val: replacement);
2069 if (!newF) {
2070 // In classic codegen, this can be a global alias, a bitcast, or a GEP.
2071 errorNYI(loc: replacement->getLoc(), feature: "replacement is not a function");
2072 continue;
2073 }
2074
2075 auto oldIt = users.find(Val: oldF.getSymNameAttr());
2076 assert(oldIt != users.end() && "replaced function was not collected");
2077 llvm::ArrayRef<mlir::Operation *> oldUsers = oldIt->second.getArrayRef();
2078 assert(verifyPointerTypeArgs(oldUsers, newF) &&
2079 "call argument types do not match replacement function");
2080
2081 // Replace old with new, but keep the old order.
2082 for (mlir::Operation *user : oldUsers)
2083 (void)mlir::SymbolTable::replaceAllSymbolUses(oldSymbol: oldF, newSymbolName: newF.getSymNameAttr(),
2084 from: user);
2085 // These users now name newF. Keep them in case newF is replaced later.
2086 auto newIt = users.find(Val: newF.getSymNameAttr());
2087 if (newIt != users.end())
2088 newIt->second.set_union(oldIt->second);
2089
2090 newF->moveBefore(existingOp: oldF);
2091 eraseGlobalSymbol(op: oldF);
2092 oldF->erase();
2093 }
2094}
2095
2096cir::GlobalOp CIRGenModule::createOrReplaceCXXRuntimeVariable(
2097 mlir::Location loc, StringRef name, mlir::Type ty,
2098 cir::GlobalLinkageKind linkage, clang::CharUnits alignment) {
2099 auto gv = mlir::dyn_cast_or_null<cir::GlobalOp>(Val: getGlobalValue(name));
2100
2101 if (gv) {
2102 // Check if the variable has the right type.
2103 if (gv.getSymType() == ty)
2104 return gv;
2105
2106 // Because of C++ name mangling, the only way we can end up with an already
2107 // existing global with the same name is if it has been declared extern
2108 // "C".
2109 assert(gv.isDeclaration() && "Declaration has wrong type!");
2110
2111 errorNYI(loc, feature: "createOrReplaceCXXRuntimeVariable: declaration exists with "
2112 "wrong type");
2113 return gv;
2114 }
2115
2116 // Create a new variable.
2117 gv = createGlobalOp(loc, name, t: ty, /*isConstant=*/true);
2118
2119 // Set up extra information and add to the module
2120 gv.setLinkageAttr(
2121 cir::GlobalLinkageKindAttr::get(context: &getMLIRContext(), value: linkage));
2122 mlir::SymbolTable::setSymbolVisibility(symbol: gv,
2123 vis: CIRGenModule::getMLIRVisibility(op: gv));
2124
2125 if (supportsCOMDAT() && cir::isWeakForLinker(linkage) &&
2126 !gv.hasAvailableExternallyLinkage()) {
2127 gv.setSelfComdat();
2128 }
2129
2130 gv.setAlignmentAttr(getSize(size: alignment));
2131 setDSOLocal(static_cast<mlir::Operation *>(gv));
2132 return gv;
2133}
2134
2135cir::GlobalLinkageKind
2136CIRGenModule::getCIRLinkageForDeclarator(const DeclaratorDecl *dd,
2137 GVALinkage linkage) {
2138 if (linkage == GVA_Internal)
2139 return cir::GlobalLinkageKind::InternalLinkage;
2140
2141 if (dd->hasAttr<WeakAttr>())
2142 return cir::GlobalLinkageKind::WeakAnyLinkage;
2143
2144 if (const auto *fd = dd->getAsFunction())
2145 if (fd->isMultiVersion() && linkage == GVA_AvailableExternally)
2146 return cir::GlobalLinkageKind::LinkOnceAnyLinkage;
2147
2148 // We are guaranteed to have a strong definition somewhere else,
2149 // so we can use available_externally linkage.
2150 if (linkage == GVA_AvailableExternally)
2151 return cir::GlobalLinkageKind::AvailableExternallyLinkage;
2152
2153 // Note that Apple's kernel linker doesn't support symbol
2154 // coalescing, so we need to avoid linkonce and weak linkages there.
2155 // Normally, this means we just map to internal, but for explicit
2156 // instantiations we'll map to external.
2157
2158 // In C++, the compiler has to emit a definition in every translation unit
2159 // that references the function. We should use linkonce_odr because
2160 // a) if all references in this translation unit are optimized away, we
2161 // don't need to codegen it. b) if the function persists, it needs to be
2162 // merged with other definitions. c) C++ has the ODR, so we know the
2163 // definition is dependable.
2164 if (linkage == GVA_DiscardableODR)
2165 return !astContext.getLangOpts().AppleKext
2166 ? cir::GlobalLinkageKind::LinkOnceODRLinkage
2167 : cir::GlobalLinkageKind::InternalLinkage;
2168
2169 // An explicit instantiation of a template has weak linkage, since
2170 // explicit instantiations can occur in multiple translation units
2171 // and must all be equivalent. However, we are not allowed to
2172 // throw away these explicit instantiations.
2173 //
2174 // CUDA/HIP: For -fno-gpu-rdc case, device code is limited to one TU,
2175 // so say that CUDA templates are either external (for kernels) or internal.
2176 // This lets llvm perform aggressive inter-procedural optimizations. For
2177 // -fgpu-rdc case, device function calls across multiple TU's are allowed,
2178 // therefore we need to follow the normal linkage paradigm.
2179 if (linkage == GVA_StrongODR) {
2180 if (getLangOpts().AppleKext)
2181 return cir::GlobalLinkageKind::ExternalLinkage;
2182 if (getLangOpts().CUDA && getLangOpts().CUDAIsDevice &&
2183 !getLangOpts().GPURelocatableDeviceCode)
2184 return dd->hasAttr<CUDAGlobalAttr>()
2185 ? cir::GlobalLinkageKind::ExternalLinkage
2186 : cir::GlobalLinkageKind::InternalLinkage;
2187 return cir::GlobalLinkageKind::WeakODRLinkage;
2188 }
2189
2190 // C++ doesn't have tentative definitions and thus cannot have common
2191 // linkage.
2192 if (!getLangOpts().CPlusPlus && isa<VarDecl>(Val: dd) &&
2193 !CodeGenUtils::isVarDeclStrongDefinition(Ctx: astContext, D: cast<VarDecl>(Val: dd),
2194 NoCommon: getCodeGenOpts().NoCommon))
2195 return cir::GlobalLinkageKind::CommonLinkage;
2196
2197 // selectany symbols are externally visible, so use weak instead of
2198 // linkonce. MSVC optimizes away references to const selectany globals, so
2199 // all definitions should be the same and ODR linkage should be used.
2200 // http://msdn.microsoft.com/en-us/library/5tkz6s71.aspx
2201 if (dd->hasAttr<SelectAnyAttr>())
2202 return cir::GlobalLinkageKind::WeakODRLinkage;
2203
2204 // Otherwise, we have strong external linkage.
2205 assert(linkage == GVA_StrongExternal);
2206 return cir::GlobalLinkageKind::ExternalLinkage;
2207}
2208
2209/// This function is called when we implement a function with no prototype, e.g.
2210/// "int foo() {}". If there are existing call uses of the old function in the
2211/// module, this adjusts them to call the new function directly.
2212///
2213/// This is not just a cleanup: the always_inline pass requires direct calls to
2214/// functions to be able to inline them. If there is a bitcast in the way, it
2215/// won't inline them. Instcombine normally deletes these calls, but it isn't
2216/// run at -O0.
2217void CIRGenModule::replaceUsesOfNonProtoTypeWithRealFunction(
2218 mlir::Operation *old, cir::FuncOp newFn) {
2219 // If we're redefining a global as a function, don't transform it.
2220 auto oldFn = mlir::dyn_cast<cir::FuncOp>(Val: old);
2221 if (!oldFn)
2222 return;
2223
2224 // TODO(cir): this RAUW ignores the features below.
2225 assert(!cir::MissingFeatures::opFuncExceptions());
2226 assert(!cir::MissingFeatures::opFuncParameterAttributes());
2227 assert(!cir::MissingFeatures::opFuncOperandBundles());
2228 // The caller sets function attributes from the replacement's declaration.
2229 // Copying the old attributes could retain incompatible assumptions from a
2230 // different declaration that happens to share the same symbol name.
2231
2232 // Mark new function as originated from a no-proto declaration.
2233 newFn.setNoProto(oldFn.getNoProto());
2234
2235 // Iterate through all calls of the no-proto function.
2236 std::optional<mlir::SymbolTable::UseRange> symUses =
2237 oldFn.getSymbolUses(from: oldFn->getParentOp());
2238
2239 if (!symUses)
2240 return;
2241
2242 for (const mlir::SymbolTable::SymbolUse &use : symUses.value()) {
2243 mlir::OpBuilder::InsertionGuard guard(builder);
2244
2245 if (auto noProtoCallOp = mlir::dyn_cast<cir::CallOp>(Val: use.getUser())) {
2246 builder.setInsertionPoint(noProtoCallOp);
2247
2248 // Patch call type with the real function type.
2249 cir::FuncType newFnType = newFn.getFunctionType();
2250 mlir::OperandRange callOperands = noProtoCallOp.getOperands();
2251 bool returnTypeMatches =
2252 newFnType.hasVoidReturn()
2253 ? noProtoCallOp.getNumResults() == 0
2254 : noProtoCallOp.getNumResults() == 1 &&
2255 noProtoCallOp.getResultTypes().front() ==
2256 newFnType.getReturnType();
2257 bool typesMatch = !newFn.getNoProto() && returnTypeMatches &&
2258 callOperands.size() == newFnType.getNumInputs();
2259 for (unsigned i = 0, e = newFnType.getNumInputs(); typesMatch && i != e;
2260 ++i) {
2261 if (callOperands[i].getType() != newFnType.getInput(i))
2262 typesMatch = false;
2263 }
2264
2265 cir::CallOp realCallOp;
2266 if (typesMatch) {
2267 // Patch call type with the real function type.
2268 realCallOp =
2269 builder.createCallOp(loc: noProtoCallOp.getLoc(), callee: newFn, operands: callOperands);
2270 } else {
2271 // Build an indirect call whose function-pointer signature matches
2272 // the existing call site. A prototyped declaration keeps its own
2273 // type, ellipsis included, so arguments passed through the ellipsis
2274 // stay variadic. A direct call to an unprototyped declaration, such
2275 // as a library call CIRGen emitted by name, keeps its own operand
2276 // types and stays non-variadic.
2277 cir::FuncType origFnType = oldFn.getFunctionType();
2278 cir::FuncType callFnType =
2279 oldFn.getNoProto()
2280 ? cir::FuncType::get(inputs: llvm::to_vector(Range: callOperands.getTypes()),
2281 returnType: origFnType.getReturnType(),
2282 /*isVarArg=*/false)
2283 : origFnType;
2284 mlir::Value addr = cir::GetGlobalOp::create(
2285 builder, location: noProtoCallOp.getLoc(), addr: cir::PointerType::get(pointee: newFnType),
2286 name: newFn.getSymName());
2287 mlir::Value casted =
2288 builder.createBitcast(src: addr, newTy: cir::PointerType::get(pointee: callFnType));
2289 realCallOp = builder.createIndirectCallOp(
2290 loc: noProtoCallOp.getLoc(), indirectTarget: casted, funcType: callFnType, operands: callOperands);
2291 }
2292 // The argument and result types are unchanged, so preserve their
2293 // attributes and the call semantics. The new callee is already set.
2294 for (mlir::NamedAttribute attr : noProtoCallOp->getAttrs())
2295 if (attr.getName() != cir::CIRDialect::getCalleeAttrName())
2296 realCallOp->setAttr(name: attr.getName(), value: attr.getValue());
2297
2298 // Replace old no proto call with fixed call.
2299 noProtoCallOp.replaceAllUsesWith(values&: realCallOp);
2300 noProtoCallOp.erase();
2301 } else if (auto getGlobalOp =
2302 mlir::dyn_cast<cir::GetGlobalOp>(Val: use.getUser())) {
2303 // The GetGlobal was emitted with the no-proto FuncType. Uses of this
2304 // operation (cir.store, cir.cast) were built for that pointer type. When
2305 // we re-type the result to the real FuncType, we need to add a bit the
2306 // old pointer type so those uses are still valid. This can lead to
2307 // some redundant bitcast chains, but those will be cleaned up by the
2308 // canonicalizer.
2309 mlir::Value res = getGlobalOp.getAddr();
2310 const mlir::Type oldResTy = res.getType();
2311 const auto newPtrTy = cir::PointerType::get(
2312 pointee: newFn.getFunctionType(),
2313 addrSpace: mlir::cast<cir::PointerType>(Val: oldResTy).getAddrSpace());
2314 if (oldResTy != newPtrTy) {
2315 res.setType(newPtrTy);
2316 builder.setInsertionPointAfter(getGlobalOp.getOperation());
2317 mlir::Value castRes =
2318 cir::CastOp::create(builder, location: getGlobalOp.getLoc(), result: oldResTy,
2319 kind: cir::CastKind::bitcast, src: res);
2320 res.replaceAllUsesExcept(newValue: castRes, exceptedUser: castRes.getDefiningOp());
2321 }
2322 } else if (mlir::isa<cir::GlobalOp>(Val: use.getUser())) {
2323 // Function addresses in global initializers use GlobalViewAttrs typed to
2324 // the initializer context (e.g. struct field type), not the FuncOp type,
2325 // so no update is required when the no-proto FuncOp is replaced.
2326 } else {
2327 llvm_unreachable(
2328 "replaceUsesOfNonProtoTypeWithRealFunction: unexpected use type");
2329 }
2330 }
2331}
2332
2333cir::GlobalLinkageKind
2334CIRGenModule::getCIRLinkageVarDefinition(const VarDecl *vd) {
2335 GVALinkage linkage = astContext.GetGVALinkageForVariable(VD: vd);
2336 return getCIRLinkageForDeclarator(dd: vd, linkage);
2337}
2338
2339cir::GlobalLinkageKind CIRGenModule::getFunctionLinkage(GlobalDecl gd) {
2340 const auto *d = cast<FunctionDecl>(Val: gd.getDecl());
2341
2342 GVALinkage linkage = astContext.GetGVALinkageForFunction(FD: d);
2343
2344 if (const auto *dtor = dyn_cast<CXXDestructorDecl>(Val: d))
2345 return getCXXABI().getCXXDestructorLinkage(linkage, dtor, dt: gd.getDtorType());
2346
2347 return getCIRLinkageForDeclarator(dd: d, linkage);
2348}
2349
2350static cir::GlobalOp
2351generateStringLiteral(mlir::Location loc, mlir::TypedAttr c,
2352 cir::GlobalLinkageKind lt, CIRGenModule &cgm,
2353 StringRef globalName, CharUnits alignment) {
2354 LangAS as = CodeGenUtils::getGlobalConstantAddressSpace(LangOpts: cgm.getLangOpts(),
2355 Target: cgm.getTarget());
2356 mlir::ptr::MemorySpaceAttrInterface addrSpace =
2357 cgm.getGlobalCIRAddressSpace(as);
2358
2359 // Create a global variable for this string
2360 // FIXME(cir): check for insertion point in module level.
2361 cir::GlobalOp gv =
2362 cgm.createGlobalOp(loc, name: globalName, t: c.getType(),
2363 isConstant: !cgm.getLangOpts().WritableStrings, addrSpace);
2364
2365 // Set up extra information and add to the module
2366 gv.setAlignmentAttr(cgm.getSize(size: alignment));
2367 gv.setLinkageAttr(
2368 cir::GlobalLinkageKindAttr::get(context: cgm.getBuilder().getContext(), value: lt));
2369 assert(!cir::MissingFeatures::opGlobalThreadLocal());
2370 gv.setUnnamedAddr(true);
2371 CIRGenModule::setInitializer(op&: gv, value: c);
2372 if (gv.isWeakForLinker()) {
2373 assert(cgm.supportsCOMDAT() && "Only COFF uses weak string literals");
2374 gv.setSelfComdat();
2375 }
2376 cgm.setDSOLocal(static_cast<mlir::Operation *>(gv));
2377 return gv;
2378}
2379
2380// LLVM IR automatically uniques names when new llvm::GlobalVariables are
2381// created. This is handy, for example, when creating globals for string
2382// literals. Since we don't do that when creating cir::GlobalOp's, we need
2383// a mechanism to generate a unique name in advance.
2384//
2385// For now, this mechanism is only used in cases where we know that the
2386// name is compiler-generated, so we don't use the MLIR symbol table for
2387// the lookup.
2388std::string CIRGenModule::getUniqueGlobalName(const std::string &baseName) {
2389 // If this is the first time we've generated a name for this basename, use
2390 // it as is and start a counter for this base name.
2391 auto it = cgGlobalNames.find(Key: baseName);
2392 if (it == cgGlobalNames.end()) {
2393 cgGlobalNames[baseName] = 1;
2394 return baseName;
2395 }
2396
2397 std::string result =
2398 baseName + "." + std::to_string(val: cgGlobalNames[baseName]++);
2399 // There should not be any symbol with this name in the module.
2400 assert(!getGlobalValue(result));
2401 return result;
2402}
2403
2404/// Return a pointer to a constant array for the given string literal.
2405cir::GlobalOp CIRGenModule::getGlobalForStringLiteral(const StringLiteral *s,
2406 StringRef name) {
2407 CharUnits alignment =
2408 astContext.getAlignOfGlobalVarInChars(T: s->getType(), /*VD=*/nullptr);
2409
2410 mlir::Attribute c = getConstantArrayFromStringLiteral(e: s);
2411
2412 cir::GlobalOp gv;
2413 if (!getLangOpts().WritableStrings && constantStringMap.count(Val: c)) {
2414 gv = constantStringMap[c];
2415 // The bigger alignment always wins.
2416 if (!gv.getAlignment() ||
2417 uint64_t(alignment.getQuantity()) > *gv.getAlignment())
2418 gv.setAlignmentAttr(getSize(size: alignment));
2419 } else {
2420 // Mangle the string literal if that's how the ABI merges duplicate strings.
2421 // Don't do it if they are writable, since we don't want writes in one TU to
2422 // affect strings in another.
2423 if (getCXXABI().getMangleContext().shouldMangleStringLiteral(SL: s) &&
2424 !getLangOpts().WritableStrings) {
2425 errorNYI(s->getSourceRange(),
2426 "getGlobalForStringLiteral: mangle string literals");
2427 }
2428
2429 // Unlike LLVM IR, CIR doesn't automatically unique names for globals, so
2430 // we need to do that explicitly.
2431 std::string uniqueName = getUniqueGlobalName(baseName: name.str());
2432 // Synthetic string literals (e.g., from SourceLocExpr) may not have valid
2433 // source locations. Use unknown location in those cases.
2434 mlir::Location loc = s->getBeginLoc().isValid()
2435 ? getLoc(cRange: s->getSourceRange())
2436 : builder.getUnknownLoc();
2437 auto typedC = llvm::cast<mlir::TypedAttr>(Val&: c);
2438 gv = generateStringLiteral(loc, c: typedC,
2439 lt: cir::GlobalLinkageKind::PrivateLinkage, cgm&: *this,
2440 globalName: uniqueName, alignment);
2441 setDSOLocal(static_cast<mlir::Operation *>(gv));
2442 constantStringMap[c] = gv;
2443
2444 assert(!cir::MissingFeatures::sanitizers());
2445 }
2446 return gv;
2447}
2448
2449/// Return a pointer to a constant array for the given string literal.
2450cir::GlobalViewAttr
2451CIRGenModule::getAddrOfConstantStringFromLiteral(const StringLiteral *s,
2452 StringRef name) {
2453 cir::GlobalOp gv = getGlobalForStringLiteral(s, name);
2454 auto arrayTy = mlir::dyn_cast<cir::ArrayType>(Val: gv.getSymType());
2455 assert(arrayTy && "String literal must be array");
2456 cir::PointerType ptrTy = getBuilder().getPointerTo(
2457 ty: arrayTy.getElementType(),
2458 as: getTypes().getPointerAddressSpace(pointeeTy: s->getType()));
2459
2460 return builder.getGlobalViewAttr(type: ptrTy, globalOp: gv);
2461}
2462
2463cir::GlobalOp CIRGenModule::createGlobalString(mlir::Location loc,
2464 StringRef str, StringRef name) {
2465 mlir::Attribute init = builder.getString(str, eltTy: convertType(type: astContext.CharTy),
2466 /*size=*/std::nullopt);
2467
2468 // Unlike LLVM IR, CIR doesn't automatically unique names for globals, so
2469 // we need to do that explicitly.
2470 return generateStringLiteral(loc, c: mlir::cast<mlir::TypedAttr>(Val&: init),
2471 lt: cir::GlobalLinkageKind::PrivateLinkage, cgm&: *this,
2472 globalName: getUniqueGlobalName(baseName: name.str()),
2473 alignment: CharUnits::One());
2474}
2475
2476mlir::ptr::MemorySpaceAttrInterface
2477CIRGenModule::getGlobalCIRAddressSpace(LangAS as) {
2478 return getTargetCIRGenInfo().getGlobalCIRAddressSpace(as);
2479}
2480
2481// TODO(cir): this could be a common AST helper for both CIR and LLVM codegen.
2482LangAS CIRGenModule::getLangTempAllocaAddressSpace() const {
2483 if (getLangOpts().OpenCL)
2484 return LangAS::opencl_private;
2485
2486 // For temporaries inside functions, CUDA treats them as normal variables.
2487 // LangAS::cuda_device, on the other hand, is reserved for those variables
2488 // explicitly marked with __device__.
2489 if (getLangOpts().CUDAIsDevice)
2490 return LangAS::Default;
2491
2492 if (getLangOpts().OpenMP && getLangOpts().OpenMPIsTargetDevice)
2493 assert(!cir::MissingFeatures::openMP());
2494
2495 if (getLangOpts().SYCLIsDevice)
2496 return LangAS::Default;
2497
2498 return LangAS::Default;
2499}
2500
2501void CIRGenModule::emitExplicitCastExprType(const ExplicitCastExpr *e,
2502 CIRGenFunction *cgf) {
2503 if (cgf && e->getType()->isVariablyModifiedType())
2504 cgf->emitVariablyModifiedType(ty: e->getType());
2505
2506 assert(!cir::MissingFeatures::generateDebugInfo() &&
2507 "emitExplicitCastExprType");
2508}
2509
2510mlir::TypedAttr CIRGenModule::emitNullMemberAttr(QualType destTy,
2511 const MemberPointerType *mpt) {
2512 if (mpt->isMemberFunctionPointerType()) {
2513 auto ty = mlir::cast<cir::MethodType>(Val: convertType(type: destTy));
2514 return builder.getNullMethodAttr(ty);
2515 }
2516
2517 auto ty = mlir::cast<cir::DataMemberType>(Val: convertType(type: destTy));
2518 return builder.getNullDataMemberAttr(ty);
2519}
2520
2521mlir::Value CIRGenModule::emitMemberPointerConstant(const UnaryOperator *e) {
2522 assert(!cir::MissingFeatures::cxxABI());
2523
2524 mlir::Location loc = getLoc(cRange: e->getSourceRange());
2525
2526 const ValueDecl *decl = cast<DeclRefExpr>(Val: e->getSubExpr())->getDecl();
2527
2528 // A member function pointer.
2529 if (const auto *methodDecl = dyn_cast<CXXMethodDecl>(Val: decl)) {
2530 auto ty = mlir::cast<cir::MethodType>(Val: convertType(type: e->getType()));
2531 if (methodDecl->isVirtual())
2532 return cir::ConstantOp::create(
2533 builder, location: loc, value: getCXXABI().buildVirtualMethodAttr(methodTy: ty, md: methodDecl));
2534
2535 const CIRGenFunctionInfo &fi =
2536 getTypes().arrangeCXXMethodDeclaration(md: methodDecl);
2537 cir::FuncType funcTy = getTypes().getFunctionType(info: fi);
2538 cir::FuncOp methodFuncOp = getAddrOfFunction(gd: methodDecl, funcType: funcTy);
2539 return cir::ConstantOp::create(builder, location: loc,
2540 value: builder.getMethodAttr(ty, methodFuncOp));
2541 }
2542
2543 // Otherwise, a member data pointer.
2544 auto ty = mlir::cast<cir::DataMemberType>(Val: convertType(type: e->getType()));
2545 const auto *mpt = e->getType()->castAs<MemberPointerType>();
2546 const auto *destClass = mpt->getMostRecentCXXRecordDecl();
2547
2548 // Empty [[no_unique_address]] fields have no CIR field index; represent the
2549 // pointer-to-data-member by its concrete byte offset within the class.
2550 if (const auto *fieldDecl = dyn_cast<FieldDecl>(Val: decl);
2551 fieldDecl && isEmptyFieldForMemberPointer(field: fieldDecl)) {
2552 // This function should ONLY be accessed in reference to itself, I don't see
2553 // any cases/couldn't find any cases where anything else could get here, and
2554 // classic-codegen does the same.
2555 assert(fieldDecl->getParent() == destClass &&
2556 "scalar member pointer should be relative to the declaring class");
2557 uint64_t offset =
2558 astContext.toCharUnitsFromBits(BitSize: astContext.getFieldOffset(FD: fieldDecl))
2559 .getQuantity();
2560 return cir::ConstantOp::create(builder, location: loc,
2561 value: cir::DataMemberOffsetAttr::get(type: ty, offset));
2562 }
2563
2564 std::optional<llvm::SmallVector<int32_t>> path =
2565 buildMemberPath(destClass, decl);
2566 if (!path)
2567 return {};
2568 return cir::ConstantOp::create(builder, location: loc,
2569 value: builder.getDataMemberAttr(ty, path: *path));
2570}
2571
2572std::optional<llvm::SmallVector<int32_t>>
2573CIRGenModule::buildMemberPath(const CXXRecordDecl *destClass,
2574 const ValueDecl *decl) {
2575 llvm::SmallVector<int32_t> path;
2576
2577 // Members of an anonymous struct/union have an IndirectFieldDecl, which
2578 // contains the whole chain of how to get to it, so to get the 'path', we dig
2579 // through those rather than searching.
2580 if (const auto *indirectField = dyn_cast<IndirectFieldDecl>(Val: decl)) {
2581 const CXXRecordDecl *currentClass = destClass;
2582 for (const NamedDecl *nd : indirectField->chain()) {
2583 const auto *field = cast<FieldDecl>(Val: nd);
2584 if (!findFieldMemberPath(currentClass, field, path))
2585 return std::nullopt;
2586 currentClass = field->getType()->getAsCXXRecordDecl();
2587 }
2588 return path;
2589 }
2590
2591 if (!findFieldMemberPath(currentClass: destClass, field: cast<FieldDecl>(Val: decl), path))
2592 return std::nullopt;
2593 return path;
2594}
2595
2596bool CIRGenModule::findFieldMemberPath(const CXXRecordDecl *currentClass,
2597 const FieldDecl *field,
2598 llvm::SmallVectorImpl<int32_t> &path) {
2599 const CIRGenRecordLayout &layout =
2600 getTypes().getCIRGenRecordLayout(rd: currentClass);
2601
2602 // The field is declared directly in this class.
2603 if (astContext.isSameEntity(X: field->getParent()->getMostRecentDecl(),
2604 Y: currentClass->getMostRecentDecl())) {
2605 int32_t fieldIdx;
2606 if (currentClass->isUnion()) {
2607 // For unions, getCIRFieldNo always returns 0 for every union member (all
2608 // members share offset 0 in the CIR record). Use the declaration-order
2609 // index to distinguish members with the same type at the same offset.
2610 if (!layout.isZeroInitializable()) {
2611 errorNYI(field->getLocation(),
2612 "data member pointer for non-zero-initializable union");
2613 return false;
2614 }
2615 fieldIdx = static_cast<int32_t>(field->getFieldIndex());
2616 } else {
2617 fieldIdx = static_cast<int32_t>(layout.getCIRFieldNo(fd: field));
2618 }
2619 path.push_back(Elt: fieldIdx);
2620 return true;
2621 }
2622
2623 // Otherwise search the base subobjects. A virtual base only blocks lowering
2624 // when the field actually lives within it; a virtual base elsewhere in the
2625 // hierarchy must not stop us from reaching a member through a non-virtual
2626 // path.
2627 for (const CXXBaseSpecifier &base : currentClass->bases()) {
2628 const auto *baseDecl =
2629 cast<CXXRecordDecl>(Val: base.getType()->getAsRecordDecl());
2630
2631 if (base.isVirtual()) {
2632 // A pointer to a data member that traverses a virtual base is ill-formed,
2633 // so this guard only fires defensively if the member is reached through
2634 // the virtual base. An unrelated virtual base is skipped so it does not
2635 // block members reached through a non-virtual path.
2636 llvm::SmallVector<int32_t> discardedPath;
2637 if (findFieldMemberPath(currentClass: baseDecl, field, path&: discardedPath)) {
2638 errorNYI(field->getLocation(),
2639 "data member pointer through virtual base");
2640 return false;
2641 }
2642 continue;
2643 }
2644
2645 // If a base class doesn't participate in layout, the field cannot be in it,
2646 // skip it.
2647 if (!layout.hasNonVirtualBaseCIRField(rd: baseDecl))
2648 continue;
2649
2650 auto baseFieldIdx =
2651 static_cast<int32_t>(layout.getNonVirtualBaseCIRFieldNo(rd: baseDecl));
2652 path.push_back(Elt: baseFieldIdx);
2653 if (findFieldMemberPath(currentClass: baseDecl, field, path))
2654 return true;
2655 path.pop_back();
2656 }
2657 return false;
2658}
2659
2660bool CIRGenModule::isEmptyFieldForMemberPointer(const FieldDecl *field) {
2661 if (!field->isPotentiallyOverlapping() ||
2662 !CodeGenUtils::isEmptyFieldForLayout(Ctx: astContext, FD: field))
2663 return false;
2664
2665 // Unions always have a field even if they are empty.
2666 const RecordDecl *rec = field->getParent();
2667 if (rec->isUnion())
2668 return true;
2669
2670 // Otherwise, count on whether accumulateFields gave this a member.
2671 return !getTypes().getCIRGenRecordLayout(rd: rec).hasCIRField(fd: field);
2672}
2673
2674void CIRGenModule::emitDeclContext(const DeclContext *dc) {
2675 for (Decl *decl : dc->decls()) {
2676 // Unlike other DeclContexts, the contents of an ObjCImplDecl at TU scope
2677 // are themselves considered "top-level", so EmitTopLevelDecl on an
2678 // ObjCImplDecl does not recursively visit them. We need to do that in
2679 // case they're nested inside another construct (LinkageSpecDecl /
2680 // ExportDecl) that does stop them from being considered "top-level".
2681 if (auto *oid = dyn_cast<ObjCImplDecl>(Val: decl))
2682 errorNYI(oid->getSourceRange(), "emitDeclConext: ObjCImplDecl");
2683
2684 emitTopLevelDecl(decl);
2685 }
2686}
2687
2688// Emit code for a single top level declaration.
2689void CIRGenModule::emitTopLevelDecl(Decl *decl) {
2690
2691 // Ignore dependent declarations.
2692 if (decl->isTemplated())
2693 return;
2694
2695 switch (decl->getKind()) {
2696 default:
2697 errorNYI(loc: decl->getBeginLoc(), feature: "declaration of kind",
2698 name: decl->getDeclKindName());
2699 break;
2700
2701 case Decl::CXXConversion:
2702 case Decl::CXXMethod:
2703 case Decl::Function: {
2704 auto *fd = cast<FunctionDecl>(Val: decl);
2705 // Consteval functions shouldn't be emitted.
2706 if (!fd->isConsteval())
2707 emitGlobal(gd: fd);
2708 break;
2709 }
2710
2711 case Decl::Import: {
2712 auto *importDecl = cast<ImportDecl>(Val: decl);
2713 clang::Module *mod = importDecl->getImportedModule();
2714
2715 // Nothing to record or to emit without a module.
2716 if (!mod)
2717 break;
2718
2719 // If we've already imported this module, we're done.
2720 if (!importedModules.insert(X: mod))
2721 break;
2722 moduleImportLocs[mod] = importDecl->getLocation();
2723
2724 assert(!cir::MissingFeatures::generateDebugInfo());
2725
2726 // A C++20 named module has its own initializer function, so its
2727 // initializers are not emitted here. The global init function of this
2728 // translation unit calls that initializer, which in turn covers the
2729 // modules it imports itself; release() records the modules to call.
2730 if (cxx20ModuleInits && mod->isNamedModule())
2731 break;
2732
2733 // For clang C++ module map modules the initializers for sub-modules are
2734 // emitted here.
2735
2736 // Find all of the submodules and emit the module initializers.
2737 llvm::SmallPtrSet<clang::Module *, 16> visited;
2738 SmallVector<clang::Module *, 16> stack;
2739 visited.insert(Ptr: mod);
2740 stack.push_back(Elt: mod);
2741
2742 while (!stack.empty()) {
2743 clang::Module *cur = stack.pop_back_val();
2744 if (!emittedModuleInitializers.insert(Ptr: cur).second)
2745 continue;
2746
2747 for (Decl *initializer : astContext.getModuleInitializers(M: cur))
2748 emitTopLevelDecl(decl: initializer);
2749
2750 // Visit the submodules of this module.
2751 for (clang::Module *submodule : cur->submodules()) {
2752 // Skip explicit children; they need to be explicitly imported to emit
2753 // the initializers.
2754 if (submodule->IsExplicit)
2755 continue;
2756
2757 if (visited.insert(Ptr: submodule).second)
2758 stack.push_back(Elt: submodule);
2759 }
2760 }
2761 break;
2762 }
2763
2764 case Decl::Export:
2765 emitDeclContext(dc: cast<ExportDecl>(Val: decl));
2766 break;
2767
2768 case Decl::Var:
2769 case Decl::Decomposition:
2770 case Decl::VarTemplateSpecialization: {
2771 emitGlobal(gd: cast<VarDecl>(Val: decl));
2772 if (auto *decomp = dyn_cast<DecompositionDecl>(Val: decl))
2773 for (auto *binding : decomp->flat_bindings())
2774 if (auto *holdingVar = binding->getHoldingVar())
2775 emitGlobal(gd: holdingVar);
2776 break;
2777 }
2778 case Decl::OpenACCRoutine:
2779 emitGlobalOpenACCRoutineDecl(cd: cast<OpenACCRoutineDecl>(Val: decl));
2780 break;
2781 case Decl::OpenACCDeclare:
2782 emitGlobalOpenACCDeclareDecl(cd: cast<OpenACCDeclareDecl>(Val: decl));
2783 break;
2784 case Decl::OMPThreadPrivate:
2785 emitOMPThreadPrivateDecl(d: cast<OMPThreadPrivateDecl>(Val: decl));
2786 break;
2787 case Decl::OMPGroupPrivate:
2788 emitOMPGroupPrivateDecl(d: cast<OMPGroupPrivateDecl>(Val: decl));
2789 break;
2790 case Decl::OMPAllocate:
2791 emitOMPAllocateDecl(d: cast<OMPAllocateDecl>(Val: decl));
2792 break;
2793 case Decl::OMPCapturedExpr:
2794 emitOMPCapturedExpr(d: cast<OMPCapturedExprDecl>(Val: decl));
2795 break;
2796 case Decl::OMPDeclareReduction:
2797 emitOMPDeclareReduction(d: cast<OMPDeclareReductionDecl>(Val: decl));
2798 break;
2799 case Decl::OMPDeclareMapper:
2800 emitOMPDeclareMapper(d: cast<OMPDeclareMapperDecl>(Val: decl));
2801 break;
2802 case Decl::OMPRequires:
2803 emitOMPRequiresDecl(d: cast<OMPRequiresDecl>(Val: decl));
2804 break;
2805 case Decl::Enum:
2806 case Decl::Using: // using X; [C++]
2807 case Decl::UsingDirective: // using namespace X; [C++]
2808 case Decl::UsingEnum: // using enum X; [C++]
2809 case Decl::NamespaceAlias:
2810 case Decl::Typedef:
2811 case Decl::TypeAlias: // using foo = bar; [C++11]
2812 case Decl::Record:
2813 assert(!cir::MissingFeatures::generateDebugInfo());
2814 break;
2815
2816 // Indirect fields from global anonymous structs and unions can be
2817 // ignored; only the actual variable requires IR gen support.
2818 case Decl::IndirectField:
2819 break;
2820
2821 // No code generation needed.
2822 case Decl::ClassTemplate:
2823 case Decl::Concept:
2824 case Decl::CXXDeductionGuide:
2825 case Decl::Empty:
2826 case Decl::ExplicitInstantiation:
2827 case Decl::FunctionTemplate:
2828 case Decl::StaticAssert:
2829 case Decl::TypeAliasTemplate:
2830 case Decl::UsingShadow:
2831 case Decl::VarTemplate:
2832 case Decl::VarTemplatePartialSpecialization:
2833 break;
2834
2835 case Decl::CXXConstructor:
2836 getCXXABI().emitCXXConstructors(d: cast<CXXConstructorDecl>(Val: decl));
2837 break;
2838 case Decl::CXXDestructor:
2839 getCXXABI().emitCXXDestructors(d: cast<CXXDestructorDecl>(Val: decl));
2840 break;
2841
2842 // C++ Decls
2843 case Decl::LinkageSpec:
2844 case Decl::Namespace:
2845 emitDeclContext(dc: Decl::castToDeclContext(decl));
2846 break;
2847
2848 case Decl::ClassTemplateSpecialization:
2849 case Decl::CXXRecord: {
2850 CXXRecordDecl *crd = cast<CXXRecordDecl>(Val: decl);
2851 assert(!cir::MissingFeatures::generateDebugInfo());
2852 for (auto *childDecl : crd->decls())
2853 if (isa<VarDecl, CXXRecordDecl, EnumDecl, OpenACCDeclareDecl>(Val: childDecl))
2854 emitTopLevelDecl(decl: childDecl);
2855 break;
2856 }
2857
2858 case Decl::FileScopeAsm:
2859 // File-scope asm is ignored during device-side CUDA compilation.
2860 if (langOpts.CUDA && langOpts.CUDAIsDevice)
2861 break;
2862 // File-scope asm is ignored during device-side OpenMP compilation.
2863 if (langOpts.OpenMPIsTargetDevice)
2864 break;
2865 // File-scope asm is ignored during device-side SYCL compilation.
2866 if (langOpts.SYCLIsDevice)
2867 break;
2868 auto *file_asm = cast<FileScopeAsmDecl>(Val: decl);
2869 std::string line = file_asm->getAsmString();
2870 globalScopeAsm.push_back(Elt: builder.getStringAttr(bytes: line));
2871 break;
2872 }
2873}
2874
2875void CIRGenModule::setInitializer(cir::GlobalOp &op, mlir::Attribute value) {
2876 // Recompute visibility when updating initializer.
2877 op.setInitialValueAttr(value);
2878 assert(!cir::MissingFeatures::opGlobalVisibility());
2879}
2880
2881std::pair<cir::FuncType, cir::FuncOp> CIRGenModule::getAddrAndTypeOfCXXStructor(
2882 GlobalDecl gd, const CIRGenFunctionInfo *fnInfo, cir::FuncType fnType,
2883 bool dontDefer, ForDefinition_t isForDefinition) {
2884 auto *md = cast<CXXMethodDecl>(Val: gd.getDecl());
2885
2886 if (isa<CXXDestructorDecl>(Val: md)) {
2887 // Always alias equivalent complete destructors to base destructors in the
2888 // MS ABI.
2889 if (getTarget().getCXXABI().isMicrosoft() &&
2890 gd.getDtorType() == Dtor_Complete &&
2891 md->getParent()->getNumVBases() == 0)
2892 errorNYI(md->getSourceRange(),
2893 "getAddrAndTypeOfCXXStructor: MS ABI complete destructor");
2894 }
2895
2896 if (!fnType) {
2897 if (!fnInfo)
2898 fnInfo = &getTypes().arrangeCXXStructorDeclaration(gd);
2899 fnType = getTypes().getFunctionType(info: *fnInfo);
2900 }
2901
2902 auto fn = getOrCreateCIRFunction(mangledName: getMangledName(gd), funcType: fnType, gd,
2903 /*ForVtable=*/forVTable: false, dontDefer,
2904 /*IsThunk=*/isThunk: false, isForDefinition);
2905
2906 return {fnType, fn};
2907}
2908
2909cir::FuncOp CIRGenModule::getAddrOfFunction(clang::GlobalDecl gd,
2910 mlir::Type funcType, bool forVTable,
2911 bool dontDefer,
2912 ForDefinition_t isForDefinition) {
2913 assert(!cast<FunctionDecl>(gd.getDecl())->isConsteval() &&
2914 "consteval function should never be emitted");
2915
2916 if (!funcType) {
2917 const auto *fd = cast<FunctionDecl>(Val: gd.getDecl());
2918 funcType = convertType(type: fd->getType());
2919 }
2920
2921 // Devirtualized destructor calls may come through here instead of via
2922 // getAddrOfCXXStructor. Make sure we use the MS ABI base destructor instead
2923 // of the complete destructor when necessary.
2924 if (const auto *dd = dyn_cast<CXXDestructorDecl>(Val: gd.getDecl())) {
2925 if (getTarget().getCXXABI().isMicrosoft() &&
2926 gd.getDtorType() == Dtor_Complete &&
2927 dd->getParent()->getNumVBases() == 0)
2928 errorNYI(dd->getSourceRange(),
2929 "getAddrOfFunction: MS ABI complete destructor");
2930 }
2931
2932 StringRef mangledName = getMangledName(gd);
2933 cir::FuncOp func =
2934 getOrCreateCIRFunction(mangledName, funcType, gd, forVTable, dontDefer,
2935 /*isThunk=*/false, isForDefinition);
2936 // Returns kernel handle for HIP kernel stub function.
2937 if (langOpts.CUDA && !langOpts.CUDAIsDevice &&
2938 cast<FunctionDecl>(Val: gd.getDecl())->hasAttr<CUDAGlobalAttr>()) {
2939 mlir::Operation *handle = getCUDARuntime().getKernelHandle(fn: func, gd);
2940
2941 // For HIP the kernel handle is a GlobalOp, which cannot be cast to
2942 // FuncOp. Return the stub directly in that case.
2943 bool isHIPHandle = mlir::isa<cir::GlobalOp>(Val: *handle);
2944 if (isForDefinition || isHIPHandle)
2945 return func;
2946 return mlir::dyn_cast<cir::FuncOp>(Val&: *handle);
2947 }
2948 return func;
2949}
2950
2951static std::string getMangledNameImpl(CIRGenModule &cgm, GlobalDecl gd,
2952 const NamedDecl *nd) {
2953 SmallString<256> buffer;
2954
2955 llvm::raw_svector_ostream out(buffer);
2956 MangleContext &mc = cgm.getCXXABI().getMangleContext();
2957
2958 assert(!cir::MissingFeatures::moduleNameHash());
2959
2960 if (mc.shouldMangleDeclName(D: nd)) {
2961 mc.mangleName(GD: gd.getWithDecl(D: nd), out);
2962 } else {
2963 IdentifierInfo *ii = nd->getIdentifier();
2964 assert(ii && "Attempt to mangle unnamed decl.");
2965
2966 const auto *fd = dyn_cast<FunctionDecl>(Val: nd);
2967 if (fd &&
2968 fd->getType()->castAs<FunctionType>()->getCallConv() == CC_X86RegCall) {
2969 cgm.errorNYI(nd->getSourceRange(), "getMangledName: X86RegCall");
2970 } else if (fd && fd->hasAttr<CUDAGlobalAttr>() &&
2971 gd.getKernelReferenceKind() == KernelReferenceKind::Stub) {
2972 out << "__device_stub__" << ii->getName();
2973 } else if (fd &&
2974 DeviceKernelAttr::isOpenCLSpelling(
2975 A: fd->getAttr<DeviceKernelAttr>()) &&
2976 gd.getKernelReferenceKind() == KernelReferenceKind::Stub) {
2977 cgm.errorNYI(nd->getSourceRange(), "getMangledName: OpenCL Stub");
2978 } else {
2979 out << ii->getName();
2980 }
2981 }
2982
2983 // Check if the module name hash should be appended for internal linkage
2984 // symbols. This should come before multi-version target suffixes are
2985 // appendded. This is to keep the name and module hash suffix of the internal
2986 // linkage function together. The unique suffix should only be added when name
2987 // mangling is done to make sure that the final name can be properly
2988 // demangled. For example, for C functions without prototypes, name mangling
2989 // is not done and the unique suffix should not be appended then.
2990 assert(!cir::MissingFeatures::moduleNameHash());
2991
2992 if (const auto *fd = dyn_cast<FunctionDecl>(Val: nd)) {
2993 if (fd->isMultiVersion()) {
2994 cgm.errorNYI(nd->getSourceRange(),
2995 "getMangledName: multi-version functions");
2996 }
2997 }
2998 // Make unique name for device side static file-scope variable for HIP.
2999 if (cgm.getASTContext().shouldExternalize(D: nd) &&
3000 cgm.getLangOpts().GPURelocatableDeviceCode)
3001 if (cgm.getLangOpts().CUDA)
3002 cgm.errorNYI(nd->getSourceRange(),
3003 "getMangledName: GPU relocatable device code");
3004
3005 return std::string(out.str());
3006}
3007
3008static FunctionDecl *
3009createOpenACCBindTempFunction(ASTContext &ctx, const IdentifierInfo *bindName,
3010 const FunctionDecl *protoFunc) {
3011 // If this is a C no-prototype function, we can take the 'easy' way out and
3012 // just create a function with no arguments/functions, etc.
3013 if (!protoFunc->hasPrototype())
3014 return FunctionDecl::Create(
3015 C&: ctx, /*DC=*/ctx.getTranslationUnitDecl(),
3016 /*StartLoc=*/SourceLocation{}, /*NLoc=*/SourceLocation{}, N: bindName,
3017 T: protoFunc->getType(), /*TInfo=*/nullptr, SC: StorageClass::SC_None);
3018
3019 QualType funcTy = protoFunc->getType();
3020 auto *fpt = cast<FunctionProtoType>(Val: protoFunc->getType());
3021
3022 // If this is a member function, add an explicit 'this' to the function type.
3023 if (auto *methodDecl = dyn_cast<CXXMethodDecl>(Val: protoFunc);
3024 methodDecl && methodDecl->isImplicitObjectMemberFunction()) {
3025 llvm::SmallVector<QualType> paramTypes{fpt->getParamTypes()};
3026 paramTypes.insert(I: paramTypes.begin(), Elt: methodDecl->getThisType());
3027
3028 funcTy = ctx.getFunctionType(ResultTy: fpt->getReturnType(), Args: paramTypes,
3029 EPI: fpt->getExtProtoInfo());
3030 fpt = cast<FunctionProtoType>(Val&: funcTy);
3031 }
3032
3033 auto *tempFunc =
3034 FunctionDecl::Create(C&: ctx, /*DC=*/ctx.getTranslationUnitDecl(),
3035 /*StartLoc=*/SourceLocation{},
3036 /*NLoc=*/SourceLocation{}, N: bindName, T: funcTy,
3037 /*TInfo=*/nullptr, SC: StorageClass::SC_None);
3038
3039 SmallVector<ParmVarDecl *> params;
3040 params.reserve(N: fpt->getNumParams());
3041
3042 // Add all of the parameters.
3043 for (unsigned i = 0, e = fpt->getNumParams(); i != e; ++i) {
3044 ParmVarDecl *parm = ParmVarDecl::Create(
3045 C&: ctx, DC: tempFunc, /*StartLoc=*/SourceLocation{},
3046 /*IdLoc=*/SourceLocation{},
3047 /*Id=*/nullptr, T: fpt->getParamType(i), /*TInfo=*/nullptr,
3048 S: StorageClass::SC_None, /*DefArg=*/nullptr);
3049 parm->setScopeInfo(scopeDepth: 0, parameterIndex: i);
3050 params.push_back(Elt: parm);
3051 }
3052
3053 tempFunc->setParams(params);
3054
3055 return tempFunc;
3056}
3057
3058std::string
3059CIRGenModule::getOpenACCBindMangledName(const IdentifierInfo *bindName,
3060 const FunctionDecl *attachedFunction) {
3061 FunctionDecl *tempFunc = createOpenACCBindTempFunction(
3062 ctx&: getASTContext(), bindName, protoFunc: attachedFunction);
3063
3064 std::string ret = getMangledNameImpl(cgm&: *this, gd: GlobalDecl(tempFunc), nd: tempFunc);
3065
3066 // This does nothing (it is a do-nothing function), since this is a
3067 // slab-allocator, but leave a call in to immediately destroy this in case we
3068 // ever come up with a way of getting allocations back.
3069 getASTContext().Deallocate(Ptr: tempFunc);
3070 return ret;
3071}
3072
3073StringRef CIRGenModule::getMangledName(GlobalDecl gd) {
3074 GlobalDecl canonicalGd = gd.getCanonicalDecl();
3075
3076 // Some ABIs don't have constructor variants. Make sure that base and complete
3077 // constructors get mangled the same.
3078 if (const auto *cd = dyn_cast<CXXConstructorDecl>(Val: canonicalGd.getDecl())) {
3079 if (!getTarget().getCXXABI().hasConstructorVariants()) {
3080 errorNYI(cd->getSourceRange(),
3081 "getMangledName: C++ constructor without variants");
3082 return cast<NamedDecl>(Val: gd.getDecl())->getIdentifier()->getName();
3083 }
3084 }
3085
3086 // In CUDA/HIP device compilation with -fgpu-rdc, the mangled name of a
3087 // static device variable depends on whether the variable is referenced by
3088 // a host or device host function. Therefore the mangled name cannot be
3089 // cached.
3090 if (!langOpts.CUDAIsDevice || !astContext.mayExternalize(D: gd.getDecl())) {
3091 auto foundName = mangledDeclNames.find(Key: canonicalGd);
3092 if (foundName != mangledDeclNames.end())
3093 return foundName->second;
3094 }
3095
3096 // Keep the first result in the case of a mangling collision.
3097 const auto *nd = cast<NamedDecl>(Val: gd.getDecl());
3098 std::string mangledName = getMangledNameImpl(cgm&: *this, gd, nd);
3099
3100 auto result = manglings.insert(KV: std::make_pair(x&: mangledName, y&: gd));
3101 return mangledDeclNames[canonicalGd] = result.first->first();
3102}
3103
3104void CIRGenModule::emitTentativeDefinition(const VarDecl *d) {
3105 assert(!d->getInit() && "Cannot emit definite definitions here!");
3106
3107 StringRef mangledName = getMangledName(gd: d);
3108 mlir::Operation *gv = getGlobalValue(name: mangledName);
3109
3110 // If we already have a definition, not declaration, with the same mangled
3111 // name, emitting of declaration is not required (and would actually overwrite
3112 // the emitted definition).
3113 if (gv) {
3114 auto gvOp = dyn_cast<cir::GlobalOp>(Val: gv);
3115 if (!gvOp) {
3116 errorNYI(d->getSourceRange(),
3117 "emitTentativeDefinition: name already used by a non-global");
3118 return;
3119 }
3120 if (!gvOp.isDeclaration())
3121 return;
3122 }
3123
3124 // If we have not seen a reference to this variable yet, place it into the
3125 // deferred declarations table to be emitted if needed later.
3126 if (!mustBeEmitted(d) && !gv) {
3127 deferredDecls[mangledName] = d;
3128 return;
3129 }
3130
3131 // The tentative definition is the only definition.
3132 emitGlobalVarDefinition(vd: d);
3133}
3134
3135bool CIRGenModule::mustBeEmitted(const ValueDecl *global) {
3136 // Never defer when EmitAllDecls is specified.
3137 if (langOpts.EmitAllDecls)
3138 return true;
3139
3140 const auto *vd = dyn_cast<VarDecl>(Val: global);
3141 if (vd &&
3142 ((codeGenOpts.KeepPersistentStorageVariables &&
3143 (vd->getStorageDuration() == SD_Static ||
3144 vd->getStorageDuration() == SD_Thread)) ||
3145 (codeGenOpts.KeepStaticConsts && vd->getStorageDuration() == SD_Static &&
3146 vd->getType().isConstQualified())))
3147 return true;
3148
3149 return getASTContext().DeclMustBeEmitted(D: global);
3150}
3151
3152bool CIRGenModule::mayBeEmittedEagerly(const ValueDecl *global) {
3153 // In OpenMP 5.0 variables and function may be marked as
3154 // device_type(host/nohost) and we should not emit them eagerly unless we sure
3155 // that they must be emitted on the host/device. To be sure we need to have
3156 // seen a declare target with an explicit mentioning of the function, we know
3157 // we have if the level of the declare target attribute is -1. Note that we
3158 // check somewhere else if we should emit this at all.
3159 if (langOpts.OpenMP >= 50 && !langOpts.OpenMPSimd) {
3160 std::optional<OMPDeclareTargetDeclAttr *> activeAttr =
3161 OMPDeclareTargetDeclAttr::getActiveAttr(VD: global);
3162 if (!activeAttr || (*activeAttr)->getLevel() != (unsigned)-1)
3163 return false;
3164 }
3165
3166 const auto *fd = dyn_cast<FunctionDecl>(Val: global);
3167 if (fd) {
3168 // Implicit template instantiations may change linkage if they are later
3169 // explicitly instantiated, so they should not be emitted eagerly.
3170 if (fd->getTemplateSpecializationKind() == TSK_ImplicitInstantiation)
3171 return false;
3172 // Defer until all versions have been semantically checked.
3173 if (fd->hasAttr<TargetVersionAttr>() && !fd->isMultiVersion())
3174 return false;
3175 // Defer emission of SYCL kernel entry point functions during device
3176 // compilation.
3177 if (langOpts.SYCLIsDevice && fd->hasAttr<SYCLKernelEntryPointAttr>())
3178 return false;
3179 }
3180 const auto *vd = dyn_cast<VarDecl>(Val: global);
3181 if (vd)
3182 if (astContext.getInlineVariableDefinitionKind(VD: vd) ==
3183 ASTContext::InlineVariableDefinitionKind::WeakUnknown)
3184 // A definition of an inline constexpr static data member may change
3185 // linkage later if it's redeclared outside the class.
3186 return false;
3187
3188 // If OpenMP is enabled and threadprivates must be generated like TLS, delay
3189 // codegen for global variables, because they may be marked as threadprivate.
3190 if (langOpts.OpenMP && langOpts.OpenMPUseTLS &&
3191 astContext.getTargetInfo().isTLSSupported() && isa<VarDecl>(Val: global) &&
3192 !global->getType().isConstantStorage(Ctx: astContext, ExcludeCtor: false, ExcludeDtor: false) &&
3193 !OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD: global))
3194 return false;
3195
3196 assert((fd || vd) &&
3197 "Only FunctionDecl and VarDecl should hit this path so far.");
3198 return true;
3199}
3200
3201static bool shouldAssumeDSOLocal(const CIRGenModule &cgm,
3202 cir::CIRGlobalValueInterface gv) {
3203 if (gv.hasLocalLinkage())
3204 return true;
3205
3206 if (!gv.hasDefaultVisibility() && !gv.hasExternalWeakLinkage())
3207 return true;
3208
3209 // DLLImport explicitly marks the GV as external.
3210 // so it shouldn't be dso_local
3211 // But we don't have the info set now
3212 assert(!cir::MissingFeatures::opGlobalDLLImportExport());
3213
3214 const llvm::Triple &tt = cgm.getTriple();
3215 const CodeGenOptions &cgOpts = cgm.getCodeGenOpts();
3216 if (tt.isOSCygMing()) {
3217 // In MinGW and Cygwin, variables without DLLImport can still be
3218 // automatically imported from a DLL by the linker; don't mark variables
3219 // that potentially could come from another DLL as DSO local.
3220
3221 // With EmulatedTLS, TLS variables can be autoimported from other DLLs
3222 // (and this actually happens in the public interface of libstdc++), so
3223 // such variables can't be marked as DSO local. (Native TLS variables
3224 // can't be dllimported at all, though.)
3225 cgm.errorNYI(feature: "shouldAssumeDSOLocal: MinGW");
3226 }
3227
3228 // On COFF, don't mark 'extern_weak' symbols as DSO local. If these symbols
3229 // remain unresolved in the link, they can be resolved to zero, which is
3230 // outside the current DSO.
3231 if (tt.isOSBinFormatCOFF() && gv.hasExternalWeakLinkage())
3232 return false;
3233
3234 // Every other GV is local on COFF.
3235 // Make an exception for windows OS in the triple: Some firmware builds use
3236 // *-win32-macho triples. This (accidentally?) produced windows relocations
3237 // without GOT tables in older clang versions; Keep this behaviour.
3238 // FIXME: even thread local variables?
3239 if (tt.isOSBinFormatCOFF() || (tt.isOSWindows() && tt.isOSBinFormatMachO()))
3240 return true;
3241
3242 // Only handle COFF and ELF for now.
3243 if (!tt.isOSBinFormatELF())
3244 return false;
3245
3246 llvm::Reloc::Model rm = cgOpts.RelocationModel;
3247 const LangOptions &lOpts = cgm.getLangOpts();
3248 if (rm != llvm::Reloc::Static && !lOpts.PIE) {
3249 // On ELF, if -fno-semantic-interposition is specified and the target
3250 // supports local aliases, there will be neither CC1
3251 // -fsemantic-interposition nor -fhalf-no-semantic-interposition. Set
3252 // dso_local on the function if using a local alias is preferable (can avoid
3253 // PLT indirection).
3254 auto func = dyn_cast<cir::FuncOp>(Val: gv.getOperation());
3255 if (!func || func.getAliasee() || !gv.canBenefitFromLocalAlias())
3256 return false;
3257 return !(lOpts.SemanticInterposition || lOpts.HalfNoSemanticInterposition);
3258 }
3259
3260 // A definition cannot be preempted from an executable.
3261 if (!gv.isDeclarationForLinker())
3262 return true;
3263
3264 // Most PIC code sequences that assume that a symbol is local cannot produce a
3265 // 0 if it turns out the symbol is undefined. While this is ABI and relocation
3266 // depended, it seems worth it to handle it here.
3267 if (rm == llvm::Reloc::PIC_ && gv.hasExternalWeakLinkage())
3268 return false;
3269
3270 // PowerPC64 prefers TOC indirection to avoid copy relocations.
3271 if (tt.isPPC64())
3272 return false;
3273
3274 if (cgOpts.DirectAccessExternalData) {
3275 // If -fdirect-access-external-data (default for -fno-pic), set dso_local
3276 // for non-thread-local variables. If the symbol is not defined in the
3277 // executable, a copy relocation will be needed at link time. dso_local is
3278 // excluded for thread-local variables because they generally don't support
3279 // copy relocations.
3280 if (auto globalOp = dyn_cast<cir::GlobalOp>(Val: gv.getOperation())) {
3281 // Assume variables are not thread-local until that support is added.
3282 assert(!cir::MissingFeatures::opGlobalThreadLocal());
3283 return true;
3284 }
3285
3286 // -fno-pic sets dso_local on a function declaration to allow direct
3287 // accesses when taking its address (similar to a data symbol). If the
3288 // function is not defined in the executable, a canonical PLT entry will be
3289 // needed at link time. -fno-direct-access-external-data can avoid the
3290 // canonical PLT entry. We don't generalize this condition to -fpie/-fpic as
3291 // it could just cause trouble without providing perceptible benefits.
3292 if (isa<cir::FuncOp>(Val: gv) && !cgOpts.NoPLT && rm == llvm::Reloc::Static)
3293 return true;
3294 }
3295
3296 // If we can use copy relocations we can assume it is local.
3297
3298 // Otherwise don't assume it is local.
3299
3300 return false;
3301}
3302
3303void CIRGenModule::setGlobalVisibility(cir::CIRGlobalValueInterface gv,
3304 const NamedDecl *d) const {
3305 // Internal definitions always have default visibility.
3306 if (gv.hasLocalLinkage()) {
3307 gv.setGlobalVisibility(cir::VisibilityKind::Default);
3308 return;
3309 }
3310 if (!d)
3311 return;
3312
3313 // Set visibility for definitions, and for declarations if requested globally
3314 // or set explicitly.
3315 LinkageInfo lv = d->getLinkageAndVisibility();
3316
3317 // OpenMP declare target variables must be visible to the host so they can
3318 // be registered. We require protected visibility unless the variable has
3319 // the DT_nohost modifier and does not need to be registered.
3320 if (getASTContext().getLangOpts().OpenMP &&
3321 getASTContext().getLangOpts().OpenMPIsTargetDevice && isa<VarDecl>(Val: d) &&
3322 d->hasAttr<OMPDeclareTargetDeclAttr>() &&
3323 d->getAttr<OMPDeclareTargetDeclAttr>()->getDevType() !=
3324 OMPDeclareTargetDeclAttr::DT_NoHost &&
3325 lv.getVisibility() == HiddenVisibility) {
3326 llvm_unreachable("setGlobalVisibility: OpenMP is NYI");
3327 return;
3328 }
3329
3330 // CUDA/HIP device kernels and global variables must be visible to the host
3331 // so they can be registered / initialized. We require protected visibility
3332 // unless the user explicitly requested hidden via an attribute.
3333 if (getASTContext().getLangOpts().CUDAIsDevice &&
3334 lv.getVisibility() == HiddenVisibility && !lv.isVisibilityExplicit() &&
3335 !d->hasAttr<OMPDeclareTargetDeclAttr>()) {
3336 bool needsProtected = false;
3337 if (isa<FunctionDecl>(Val: d)) {
3338 needsProtected =
3339 d->hasAttr<CUDAGlobalAttr>() || d->hasAttr<DeviceKernelAttr>();
3340 } else if (const auto *vd = dyn_cast<VarDecl>(Val: d)) {
3341 needsProtected = vd->hasAttr<CUDADeviceAttr>() ||
3342 vd->hasAttr<CUDAConstantAttr>() ||
3343 vd->getType()->isCUDADeviceBuiltinSurfaceType() ||
3344 vd->getType()->isCUDADeviceBuiltinTextureType();
3345 }
3346 if (needsProtected) {
3347 gv.setGlobalVisibility(cir::VisibilityKind::Protected);
3348 return;
3349 }
3350 }
3351
3352 if (getASTContext().getLangOpts().HLSL && !d->isInExportDeclContext()) {
3353 gv.setGlobalVisibility(cir::VisibilityKind::Hidden);
3354 return;
3355 }
3356
3357 assert(!cir::MissingFeatures::opGlobalDLLImportExport());
3358
3359 if (lv.isVisibilityExplicit() || getLangOpts().SetVisibilityForExternDecls ||
3360 !gv.isDeclarationForLinker())
3361 gv.setGlobalVisibility(getCIRVisibilityKind(v: lv.getVisibility()));
3362}
3363
3364void CIRGenModule::setDSOLocal(cir::CIRGlobalValueInterface gv) const {
3365 gv.setDSOLocal(shouldAssumeDSOLocal(cgm: *this, gv));
3366}
3367
3368void CIRGenModule::setDSOLocal(mlir::Operation *op) const {
3369 if (auto globalValue = dyn_cast<cir::CIRGlobalValueInterface>(Val: op))
3370 setDSOLocal(globalValue);
3371}
3372
3373void CIRGenModule::setGVProperties(mlir::Operation *op,
3374 const NamedDecl *d) const {
3375 assert(!cir::MissingFeatures::opGlobalDLLImportExport());
3376 setGVPropertiesAux(op, d);
3377}
3378
3379void CIRGenModule::setGVPropertiesAux(mlir::Operation *op,
3380 const NamedDecl *d) const {
3381 setGlobalVisibility(gv: cast<cir::CIRGlobalValueInterface>(Val: op), d);
3382 setDSOLocal(op);
3383 assert(!cir::MissingFeatures::opGlobalPartition());
3384}
3385
3386bool CIRGenModule::lookupRepresentativeDecl(StringRef mangledName,
3387 GlobalDecl &result) const {
3388 auto res = manglings.find(Key: mangledName);
3389 if (res == manglings.end())
3390 return false;
3391 result = res->getValue();
3392 return true;
3393}
3394
3395static cir::TLSModel getCIRTLSModel(StringRef S) {
3396 return llvm::StringSwitch<cir::TLSModel>(S)
3397 .Case(S: "global-dynamic", Value: cir::TLSModel::GeneralDynamic)
3398 .Case(S: "local-dynamic", Value: cir::TLSModel::LocalDynamic)
3399 .Case(S: "initial-exec", Value: cir::TLSModel::InitialExec)
3400 .Case(S: "local-exec", Value: cir::TLSModel::LocalExec);
3401}
3402
3403cir::TLSModel CIRGenModule::getDefaultCIRTLSModel() const {
3404 switch (getCodeGenOpts().getDefaultTLSModel()) {
3405 case CodeGenOptions::GeneralDynamicTLSModel:
3406 return cir::TLSModel::GeneralDynamic;
3407 case CodeGenOptions::LocalDynamicTLSModel:
3408 return cir::TLSModel::LocalDynamic;
3409 case CodeGenOptions::InitialExecTLSModel:
3410 return cir::TLSModel::InitialExec;
3411 case CodeGenOptions::LocalExecTLSModel:
3412 return cir::TLSModel::LocalExec;
3413 }
3414 llvm_unreachable("Invalid TLS model!");
3415}
3416
3417void CIRGenModule::setTLSMode(mlir::Operation *op, const VarDecl &d,
3418 bool isExtendingDecl) {
3419 assert(d.getTLSKind() && "setting TLS mode on non-TLS var!");
3420
3421 cir::TLSModel tlm = getDefaultCIRTLSModel();
3422
3423 // Override the TLS model if it is explicitly specified.
3424 if (const auto *attr = d.getAttr<TLSModelAttr>())
3425 tlm = getCIRTLSModel(S: attr->getModel());
3426
3427 auto global = cast<cir::GlobalOp>(Val: op);
3428 global.setTlsModel(tlm);
3429
3430 // For namespace-scope dyanmic TLS we need to set the wrapper, int, or guard
3431 // info.
3432 if (d.isStaticLocal())
3433 return;
3434
3435 // If this function was called to set the TLS mode for a temporary whose
3436 // lifetime is extended by the variable declared by `d`, don't emit the
3437 // wrapper, init, and guard info.
3438 if (isExtendingDecl)
3439 return;
3440
3441 setGlobalTlsReferences(vd: d, globalOp: global);
3442}
3443
3444void CIRGenModule::setCIRFunctionAttributes(GlobalDecl globalDecl,
3445 const CIRGenFunctionInfo &info,
3446 cir::FuncOp func, bool isThunk) {
3447 // TODO(cir): More logic of constructAttributeList is needed.
3448 cir::CallingConv callingConv;
3449
3450 // TODO(cir): The current list should be initialized with the extra function
3451 // attributes, but we don't have those yet. For now, the PAL is initialized
3452 // with nothing.
3453 assert(!cir::MissingFeatures::opFuncExtraAttrs());
3454 // Initialize PAL with existing attributes to merge attributes.
3455 mlir::NamedAttrList pal{};
3456 std::vector<mlir::NamedAttrList> argAttrs(info.arguments().size());
3457 mlir::NamedAttrList retAttrs{};
3458 constructAttributeList(name: func.getName(), info, calleeInfo: globalDecl, attrs&: pal, argAttrs,
3459 retAttrs, callingConv,
3460 /*attrOnCallSite=*/false, isThunk);
3461
3462 for (mlir::NamedAttribute attr : pal)
3463 func->setAttr(name: attr.getName(), value: attr.getValue());
3464
3465 llvm::for_each(Range: llvm::enumerate(First&: argAttrs), F: [func](auto idx_arg_pair) {
3466 mlir::function_interface_impl::setArgAttrs(func, idx_arg_pair.index(),
3467 idx_arg_pair.value());
3468 });
3469 if (!retAttrs.empty())
3470 mlir::function_interface_impl::setResultAttrs(op: func, index: 0, attributes: retAttrs);
3471
3472 // TODO(cir): Check X86_VectorCall incompatibility wiht WinARM64EC
3473
3474 func.setCallingConv(callingConv);
3475}
3476
3477void CIRGenModule::setFunctionAttributes(GlobalDecl globalDecl,
3478 cir::FuncOp func,
3479 bool isIncompleteFunction,
3480 bool isThunk) {
3481 // NOTE(cir): Original CodeGen checks if this is an intrinsic. In CIR we
3482 // represent them in dedicated ops. The correct attributes are ensured during
3483 // translation to LLVM. Thus, we don't need to check for them here.
3484
3485 const auto *funcDecl = cast<FunctionDecl>(Val: globalDecl.getDecl());
3486
3487 if (!isIncompleteFunction)
3488 setCIRFunctionAttributes(globalDecl,
3489 info: getTypes().arrangeGlobalDeclaration(gd: globalDecl),
3490 func, isThunk);
3491
3492 // Add the Returned attribute for "this", except for iOS 5 and earlier
3493 // where substantial code, including the libstdc++ dylib, was compiled with
3494 // GCC and does not actually return "this".
3495 if (!isThunk && getCXXABI().hasThisReturn(gd: globalDecl) &&
3496 !(getTriple().isiOS() && getTriple().isOSVersionLT(Major: 6)))
3497 func.setArgAttr(index: 0, name: mlir::LLVM::LLVMDialect::getReturnedAttrName(),
3498 value: mlir::UnitAttr::get(context: &getMLIRContext()));
3499
3500 if (!isIncompleteFunction && func.isDeclaration())
3501 getTargetCIRGenInfo().setTargetAttributes(decl: funcDecl, global: func, module&: *this);
3502
3503 // Diagnose calls to this function at the backend level, mirroring
3504 // CodeGenModule::SetFunctionAttributes's "dontcall-error"/"dontcall-warn".
3505 if (const auto *errorAttr = funcDecl->getAttr<ErrorAttr>()) {
3506 if (errorAttr->isError())
3507 func->setAttr(name: cir::CIRDialect::getDontCallErrorAttrName(),
3508 value: mlir::StringAttr::get(context: &getMLIRContext(),
3509 bytes: errorAttr->getUserDiagnostic()));
3510 else if (errorAttr->isWarning())
3511 func->setAttr(name: cir::CIRDialect::getDontCallWarnAttrName(),
3512 value: mlir::StringAttr::get(context: &getMLIRContext(),
3513 bytes: errorAttr->getUserDiagnostic()));
3514 }
3515
3516 // Mirrors setLinkageForGV in CodeGenModule::SetFunctionAttributes.
3517 setLinkageForFunction(cgm&: *this, func, nd: funcDecl);
3518
3519 // If we plan on emitting this inline builtin, we can't treat it as a builtin.
3520 if (funcDecl->isInlineBuiltinDeclaration()) {
3521 const FunctionDecl *fdBody;
3522 bool hasBody = funcDecl->hasBody(Definition&: fdBody);
3523 (void)hasBody;
3524 assert(hasBody && "Inline builtin declarations should always have an "
3525 "available body!");
3526 assert(!cir::MissingFeatures::attributeNoBuiltin());
3527 }
3528
3529 if (funcDecl->isReplaceableGlobalAllocationFunction()) {
3530 // A replaceable global allocation function does not act like a builtin by
3531 // default, only if it is invoked by a new-expression or delete-expression.
3532 func->setAttr(name: cir::CIRDialect::getNoBuiltinAttrName(),
3533 value: mlir::UnitAttr::get(context: &getMLIRContext()));
3534 }
3535
3536 if (isa<CXXConstructorDecl>(Val: funcDecl) || isa<CXXDestructorDecl>(Val: funcDecl))
3537 func.setUnnamedAddr(true);
3538 else if (const auto *md = dyn_cast<CXXMethodDecl>(Val: funcDecl))
3539 if (md->isVirtual())
3540 func.setUnnamedAddr(true);
3541}
3542
3543void CIRGenModule::setCIRFunctionAttributesForDefinition(
3544 const clang::Decl *decl, cir::FuncOp f) {
3545
3546 if ((!decl || !decl->hasAttr<NoUwtableAttr>()) && codeGenOpts.UnwindTables)
3547 f.setUwtable(static_cast<cir::UnwindTableKind>(codeGenOpts.UnwindTables));
3548
3549 assert(!cir::MissingFeatures::stackProtector());
3550
3551 if (!CodeGenUtils::hasUnwindExceptions(LangOpts: langOpts))
3552 f->setAttr(name: cir::CIRDialect::getNoThrowAttrName(),
3553 value: mlir::UnitAttr::get(context: &getMLIRContext()));
3554
3555 std::optional<cir::InlineKind> existingInlineKind = f.getInlineKind();
3556 bool isNoInline =
3557 existingInlineKind && *existingInlineKind == cir::InlineKind::NoInline;
3558 bool isAlwaysInline = existingInlineKind &&
3559 *existingInlineKind == cir::InlineKind::AlwaysInline;
3560 if (!decl) {
3561 assert(!cir::MissingFeatures::hlsl());
3562
3563 if (!isAlwaysInline &&
3564 codeGenOpts.getInlining() == CodeGenOptions::OnlyAlwaysInlining) {
3565 // If inlining is disabled and we don't have a declaration to control
3566 // inlining, mark the function as 'noinline' unless it is explicitly
3567 // marked as 'alwaysinline'.
3568 f.setInlineKind(cir::InlineKind::NoInline);
3569 }
3570
3571 return;
3572 }
3573
3574 assert(!cir::MissingFeatures::opFuncArmStreamingAttr());
3575 assert(!cir::MissingFeatures::opFuncArmNewAttr());
3576 assert(!cir::MissingFeatures::hlsl());
3577
3578 // Track whether we need to add the optnone attribute, starting with the
3579 // default for this optimization level.
3580 bool shouldAddOptNone =
3581 !codeGenOpts.DisableO0ImplyOptNone && codeGenOpts.OptimizationLevel == 0;
3582 // optnone is incompatible with minsize and alwaysinline.
3583 shouldAddOptNone &= !decl->hasAttr<MinSizeAttr>();
3584 shouldAddOptNone &= !decl->hasAttr<AlwaysInlineAttr>();
3585
3586 // Handle inline attributes
3587 if ((shouldAddOptNone || decl->hasAttr<OptimizeNoneAttr>()) &&
3588 !isAlwaysInline) {
3589 // Add optnone, but do so only if the function isn't always_inline.
3590 f->setAttr(name: cir::CIRDialect::getOptNoneAttrName(),
3591 value: mlir::UnitAttr::get(context: &getMLIRContext()));
3592
3593 // OptimizeNone implies noinline; we should not be inlining such functions.
3594 f.setInlineKind(cir::InlineKind::NoInline);
3595
3596 // We still need to handle naked functions even though optnone subsumes
3597 // much of their semantics.
3598 if (decl->hasAttr<NakedAttr>())
3599 f->setAttr(name: cir::CIRDialect::getNakedAttrName(),
3600 value: mlir::UnitAttr::get(context: &getMLIRContext()));
3601
3602 // OptimizeNone wins over OptimizeForSize and MinSize.
3603 f->removeAttr(name: cir::CIRDialect::getOptimizeForSizeAttrName());
3604 f->removeAttr(name: cir::CIRDialect::getMinSizeAttrName());
3605 } else if (decl->hasAttr<NakedAttr>()) {
3606 // Naked implies noinline: we should not be inlining such functions.
3607 f->setAttr(name: cir::CIRDialect::getNakedAttrName(),
3608 value: mlir::UnitAttr::get(context: &getMLIRContext()));
3609 f.setInlineKind(cir::InlineKind::NoInline);
3610 } else if (decl->hasAttr<NoDuplicateAttr>()) {
3611 f->setAttr(name: cir::CIRDialect::getNoDuplicatesAttrName(),
3612 value: mlir::UnitAttr::get(context: &getMLIRContext()));
3613 } else if (decl->hasAttr<NoInlineAttr>() && !isAlwaysInline) {
3614 // Add noinline if the function isn't always_inline.
3615 f.setInlineKind(cir::InlineKind::NoInline);
3616 } else if (decl->hasAttr<AlwaysInlineAttr>() && !isNoInline) {
3617 // Don't override AlwaysInline with NoInline, or vice versa, since we can't
3618 // specify both in IR.
3619 f.setInlineKind(cir::InlineKind::AlwaysInline);
3620 } else if (codeGenOpts.getInlining() == CodeGenOptions::OnlyAlwaysInlining) {
3621 // If inlining is disabled, force everything that isn't always_inline
3622 // to carry an explicit noinline attribute.
3623 if (!isAlwaysInline)
3624 f.setInlineKind(cir::InlineKind::NoInline);
3625 } else {
3626 // Otherwise, propagate the inline hint attribute and potentially use its
3627 // absence to mark things as noinline.
3628 // Search function and template pattern redeclarations for inline.
3629 if (auto *fd = dyn_cast<FunctionDecl>(Val: decl)) {
3630 // TODO: Share this checkForInline implementation with classic codegen.
3631 // This logic is likely to change over time, so sharing would help ensure
3632 // consistency.
3633 auto checkForInline = [](const FunctionDecl *decl) {
3634 auto checkRedeclForInline = [](const FunctionDecl *redecl) {
3635 return redecl->isInlineSpecified();
3636 };
3637 if (any_of(Range: decl->redecls(), P: checkRedeclForInline))
3638 return true;
3639 const FunctionDecl *pattern = decl->getTemplateInstantiationPattern();
3640 if (!pattern)
3641 return false;
3642 return any_of(Range: pattern->redecls(), P: checkRedeclForInline);
3643 };
3644 if (checkForInline(fd)) {
3645 f.setInlineKind(cir::InlineKind::InlineHint);
3646 } else if (codeGenOpts.getInlining() ==
3647 CodeGenOptions::OnlyHintInlining &&
3648 !fd->isInlined() && !isAlwaysInline) {
3649 f.setInlineKind(cir::InlineKind::NoInline);
3650 }
3651 }
3652 }
3653
3654 // Add other optimization related attributes if we are optimizing this
3655 // function.
3656 if (!decl->hasAttr<OptimizeNoneAttr>()) {
3657 if (decl->hasAttr<ColdAttr>()) {
3658 if (!shouldAddOptNone)
3659 f->setAttr(name: cir::CIRDialect::getOptimizeForSizeAttrName(),
3660 value: mlir::UnitAttr::get(context: &getMLIRContext()));
3661
3662 f->setAttr(name: cir::CIRDialect::getColdAttrName(),
3663 value: mlir::UnitAttr::get(context: &getMLIRContext()));
3664 }
3665
3666 if (decl->hasAttr<HotAttr>())
3667 f->setAttr(name: cir::CIRDialect::getHotAttrName(),
3668 value: mlir::UnitAttr::get(context: &getMLIRContext()));
3669
3670 if (decl->hasAttr<MinSizeAttr>())
3671 f->setAttr(name: cir::CIRDialect::getMinSizeAttrName(),
3672 value: mlir::UnitAttr::get(context: &getMLIRContext()));
3673 }
3674
3675 std::optional<uint64_t> explicitAlignment;
3676 if (unsigned alignment =
3677 decl->getMaxAlignment() / getASTContext().getCharWidth())
3678 explicitAlignment = alignment;
3679 else if (langOpts.FunctionAlignment)
3680 explicitAlignment = 1ull << langOpts.FunctionAlignment;
3681
3682 if (explicitAlignment) {
3683 f.setAlignment(*explicitAlignment);
3684 f.setPreferredAlignment(*explicitAlignment);
3685 } else if (langOpts.PreferredFunctionAlignment) {
3686 f.setPreferredAlignment(langOpts.PreferredFunctionAlignment);
3687 }
3688
3689 // Some C++ ABIs require 2-byte alignment for member functions, in order to
3690 // reserve a bit for differentiating between virtual and non-virtual member
3691 // functions. If the current target's C++ ABI requires this and this is a
3692 // member function, set its alignment accordingly.
3693 if (getTarget().getCXXABI().areMemberFunctionsAligned()) {
3694 if (isa<CXXMethodDecl>(Val: decl) && f.getAlignment().value_or(u: 1) < 2)
3695 f.setAlignment(2);
3696 }
3697
3698 // Attach "sycl-module-id" to sycl_external function definitions to mark
3699 // them as entry points for per-translation-unit device-code splitting.
3700 if (getLangOpts().SYCLIsDevice && decl->hasAttr<SYCLExternalAttr>())
3701 addSYCLModuleIdAttr(fn: f);
3702}
3703
3704// Maps an AST address space to the OpenCL logical address space kind recorded
3705// in kernel argument metadata. This mapping is independent of the target
3706// address space map, allowing consumers to distinguish OpenCL logical address
3707// spaces even when the target maps them to the same address space.
3708static cir::LangAddressSpace
3709getOpenCLKernelArgAddressSpace(LangAS addressSpace) {
3710 switch (addressSpace) {
3711 case LangAS::opencl_global:
3712 return cir::LangAddressSpace::OffloadGlobal;
3713 case LangAS::opencl_constant:
3714 return cir::LangAddressSpace::OffloadConstant;
3715 case LangAS::opencl_local:
3716 return cir::LangAddressSpace::OffloadLocal;
3717 case LangAS::opencl_generic:
3718 return cir::LangAddressSpace::OffloadGeneric;
3719 case LangAS::opencl_global_device:
3720 return cir::LangAddressSpace::OffloadGlobalDevice;
3721 case LangAS::opencl_global_host:
3722 return cir::LangAddressSpace::OffloadGlobalHost;
3723 default:
3724 // All other AST address spaces, including target-specific ones, use the
3725 // OpenCL metadata default, which lowers to SPIR address space ID 0.
3726 return cir::LangAddressSpace::Default;
3727 }
3728}
3729
3730void CIRGenModule::emitOpenCLKernelArgMetadata(cir::FuncOp func,
3731 const clang::FunctionDecl *fd) {
3732 assert(fd && "expected a kernel function declaration");
3733 const PrintingPolicy &policy = getASTContext().getPrintingPolicy();
3734
3735 // Create arrays that represent the kernel argument metadata. Each array has
3736 // one value per kernel argument, in source order.
3737 SmallVector<mlir::Attribute> addressQuals;
3738 SmallVector<mlir::Attribute> accessQuals;
3739 SmallVector<mlir::Attribute> argTypeNames;
3740 SmallVector<mlir::Attribute> argBaseTypeNames;
3741 SmallVector<mlir::Attribute> argTypeQuals;
3742 SmallVector<mlir::Attribute> argNames;
3743
3744 for (const ParmVarDecl *param : fd->parameters()) {
3745 argNames.push_back(Elt: builder.getStringAttr(bytes: param->getName()));
3746
3747 QualType type = param->getType();
3748 std::string typeQuals;
3749
3750 if (type->isImageType() || type->isPipeType()) {
3751 errorNYI(param->getSourceRange(),
3752 "OpenCL kernel argument metadata for image and pipe types");
3753 return;
3754 }
3755
3756 accessQuals.push_back(Elt: builder.getStringAttr(bytes: "none"));
3757
3758 auto getTypeSpelling = [&](QualType paramType) {
3759 std::string typeName = paramType.getUnqualifiedType().getAsString(Policy: policy);
3760
3761 if (paramType.isCanonical()) {
3762 StringRef typeNameRef = typeName;
3763 if (typeNameRef.consume_front(Prefix: "unsigned "))
3764 return std::string("u") + typeNameRef.str();
3765 if (typeNameRef.consume_front(Prefix: "signed "))
3766 return typeNameRef.str();
3767 }
3768
3769 return typeName;
3770 };
3771
3772 // Type metadata preserves source spelling, while base type metadata uses
3773 // canonical spelling without typedefs.
3774 if (type->isPointerType()) {
3775 QualType pointeeType = type->getPointeeType();
3776 addressQuals.push_back(Elt: cir::LangAddressSpaceAttr::get(
3777 context: &getMLIRContext(),
3778 value: getOpenCLKernelArgAddressSpace(addressSpace: pointeeType.getAddressSpace())));
3779
3780 argTypeNames.push_back(
3781 Elt: builder.getStringAttr(bytes: getTypeSpelling(pointeeType) + "*"));
3782 argBaseTypeNames.push_back(Elt: builder.getStringAttr(
3783 bytes: getTypeSpelling(pointeeType.getCanonicalType()) + "*"));
3784
3785 if (type.isRestrictQualified())
3786 typeQuals = "restrict";
3787 if (pointeeType.isConstQualified() ||
3788 pointeeType.getAddressSpace() == LangAS::opencl_constant)
3789 typeQuals += typeQuals.empty() ? "const" : " const";
3790 if (pointeeType.isVolatileQualified())
3791 typeQuals += typeQuals.empty() ? "volatile" : " volatile";
3792 } else {
3793 addressQuals.push_back(Elt: cir::LangAddressSpaceAttr::get(
3794 context: &getMLIRContext(), value: cir::LangAddressSpace::Default));
3795
3796 argTypeNames.push_back(Elt: builder.getStringAttr(bytes: getTypeSpelling(type)));
3797 argBaseTypeNames.push_back(
3798 Elt: builder.getStringAttr(bytes: getTypeSpelling(type.getCanonicalType())));
3799 }
3800
3801 argTypeQuals.push_back(Elt: builder.getStringAttr(bytes: typeQuals));
3802 }
3803
3804 mlir::ArrayAttr names;
3805 if (getCodeGenOpts().EmitOpenCLArgMetadata)
3806 names = builder.getArrayAttr(value: argNames);
3807
3808 mlir::Attribute metadata = cir::OpenCLKernelArgMetadataAttr::get(
3809 context: func.getContext(), addr_space: builder.getArrayAttr(value: addressQuals),
3810 access_qual: builder.getArrayAttr(value: accessQuals), type: builder.getArrayAttr(value: argTypeNames),
3811 base_type: builder.getArrayAttr(value: argBaseTypeNames),
3812 type_qual: builder.getArrayAttr(value: argTypeQuals), name: names);
3813 func->setAttr(name: cir::CIRDialect::getOpenCLKernelArgMetadataAttrName(),
3814 value: metadata);
3815}
3816
3817cir::FuncOp CIRGenModule::getOrCreateCIRFunction(
3818 StringRef mangledName, mlir::Type funcType, GlobalDecl gd, bool forVTable,
3819 bool dontDefer, bool isThunk, ForDefinition_t isForDefinition,
3820 mlir::NamedAttrList extraAttrs) {
3821 const Decl *d = gd.getDecl();
3822
3823 if (const auto *fd = cast_or_null<FunctionDecl>(Val: d)) {
3824 // For the device, mark the function as one that should be emitted.
3825 if (getLangOpts().OpenMPIsTargetDevice && openMPRuntime &&
3826 !getOpenMPRuntime().markAsGlobalTarget(gd) && fd->isDefined() &&
3827 !dontDefer && !isForDefinition) {
3828 if (const FunctionDecl *fdDef = fd->getDefinition()) {
3829 GlobalDecl gdDef;
3830 if (const auto *cd = dyn_cast<CXXConstructorDecl>(Val: fdDef))
3831 gdDef = GlobalDecl(cd, gd.getCtorType());
3832 else if (const auto *dd = dyn_cast<CXXDestructorDecl>(Val: fdDef))
3833 gdDef = GlobalDecl(dd, gd.getDtorType());
3834 else
3835 gdDef = GlobalDecl(fdDef);
3836 emitGlobal(gd: gdDef);
3837 }
3838 }
3839
3840 // Any attempts to use a MultiVersion function should result in retrieving
3841 // the iFunc instead. Name mangling will handle the rest of the changes.
3842 if (fd->isMultiVersion())
3843 errorNYI(fd->getSourceRange(), "getOrCreateCIRFunction: multi-version");
3844 }
3845
3846 // Lookup the entry, lazily creating it if necessary.
3847 mlir::Operation *entry = getGlobalValue(name: mangledName);
3848 if (entry) {
3849 assert(mlir::isa<cir::FuncOp>(entry));
3850
3851 // A weakref's target is only weak until something else names it.
3852 if (weakRefReferences.erase(Ptr: entry)) {
3853 const auto *fd = cast_or_null<FunctionDecl>(Val: d);
3854 if (fd && !fd->hasAttr<WeakAttr>())
3855 cast<cir::FuncOp>(Val: entry).setLinkage(
3856 cir::GlobalLinkageKind::ExternalLinkage);
3857 }
3858
3859 // Handle dropped DLL attributes.
3860 if (d && !d->hasAttr<DLLImportAttr>() && !d->hasAttr<DLLExportAttr>()) {
3861 assert(!cir::MissingFeatures::setDLLStorageClass());
3862 setDSOLocal(entry);
3863 }
3864
3865 // If there are two attempts to define the same mangled name, issue an
3866 // error.
3867 auto fn = cast<cir::FuncOp>(Val: entry);
3868 if (isForDefinition && fn && !fn.isDeclaration()) {
3869 GlobalDecl otherGd;
3870 // Check that GD is not yet in DiagnosedConflictingDefinitions is required
3871 // to make sure that we issue an error only once.
3872 if (lookupRepresentativeDecl(mangledName, result&: otherGd) &&
3873 (gd.getCanonicalDecl().getDecl() !=
3874 otherGd.getCanonicalDecl().getDecl()) &&
3875 diagnosedConflictingDefinitions.insert(V: gd).second) {
3876 getDiags().Report(Loc: d->getLocation(), DiagID: diag::err_duplicate_mangled_name)
3877 << mangledName;
3878 getDiags().Report(Loc: otherGd.getDecl()->getLocation(),
3879 DiagID: diag::note_previous_definition);
3880 }
3881 }
3882
3883 if (fn && fn.getFunctionType() == funcType) {
3884 return fn;
3885 }
3886
3887 if (!isForDefinition) {
3888 return fn;
3889 }
3890
3891 // TODO(cir): classic codegen checks here if this is a llvm::GlobalAlias.
3892 // How will we support this?
3893 }
3894
3895 auto *funcDecl = llvm::cast_or_null<FunctionDecl>(Val: gd.getDecl());
3896 bool invalidLoc = !funcDecl ||
3897 funcDecl->getSourceRange().getBegin().isInvalid() ||
3898 funcDecl->getSourceRange().getEnd().isInvalid();
3899 cir::FuncOp funcOp = createCIRFunction(
3900 loc: invalidLoc ? theModule->getLoc() : getLoc(cRange: funcDecl->getSourceRange()),
3901 name: mangledName, funcType: mlir::cast<cir::FuncType>(Val&: funcType), funcDecl);
3902
3903 if (funcDecl && funcDecl->hasAttr<AnnotateAttr>())
3904 deferredAnnotations[mangledName] = funcDecl;
3905
3906 // If we already created a function with the same mangled name (but different
3907 // type) before, take its name and add it to the list of functions to be
3908 // replaced with F at the end of CodeGen.
3909 //
3910 // This happens if there is a prototype for a function (e.g. "int f()") and
3911 // then a definition of a different type (e.g. "int f(int x)").
3912 if (entry) {
3913
3914 // Fetch a generic symbol-defining operation and its uses.
3915 auto symbolOp = mlir::cast<mlir::SymbolOpInterface>(Val: entry);
3916
3917 // This might be an implementation of a function without a prototype, in
3918 // which case, try to do special replacement of calls which match the new
3919 // prototype. The really key thing here is that we also potentially drop
3920 // arguments from the call site so as to make a direct call, which makes the
3921 // inliner happier and suppresses a number of optimizer warnings (!) about
3922 // dropping arguments.
3923 if (symbolOp.getSymbolUses(from: symbolOp->getParentOp()))
3924 replaceUsesOfNonProtoTypeWithRealFunction(old: entry, newFn: funcOp);
3925
3926 // Obliterate no-proto declaration.
3927 eraseGlobalSymbol(op: entry);
3928 entry->erase();
3929 }
3930
3931 if (d)
3932 setFunctionAttributes(globalDecl: gd, func: funcOp, /*isIncompleteFunction=*/false, isThunk);
3933 if (!extraAttrs.empty())
3934 for (mlir::NamedAttribute attr : extraAttrs)
3935 if (!funcOp->hasDiscardableAttr(name: attr.getName()))
3936 funcOp->setDiscardableAttr(name: attr.getName(), value: attr.getValue());
3937
3938 // 'dontDefer' actually means don't move this to the deferredDeclsToEmit list.
3939 if (dontDefer) {
3940 // TODO(cir): This assertion will need an additional condition when we
3941 // support incomplete functions.
3942 assert(funcOp.getFunctionType() == funcType);
3943 return funcOp;
3944 }
3945
3946 // All MSVC dtors other than the base dtor are linkonce_odr and delegate to
3947 // each other bottoming out wiht the base dtor. Therefore we emit non-base
3948 // dtors on usage, even if there is no dtor definition in the TU.
3949 if (isa_and_nonnull<CXXDestructorDecl>(Val: d) &&
3950 getCXXABI().useThunkForDtorVariant(dtor: cast<CXXDestructorDecl>(Val: d),
3951 dt: gd.getDtorType()))
3952 errorNYI(d->getSourceRange(), "getOrCreateCIRFunction: dtor");
3953
3954 // This is the first use or definition of a mangled name. If there is a
3955 // deferred decl with this name, remember that we need to emit it at the end
3956 // of the file.
3957 auto ddi = deferredDecls.find(x: mangledName);
3958 if (ddi != deferredDecls.end()) {
3959 // Move the potentially referenced deferred decl to the
3960 // DeferredDeclsToEmit list, and remove it from DeferredDecls (since we
3961 // don't need it anymore).
3962 addDeferredDeclToEmit(GD: ddi->second);
3963 deferredDecls.erase(position: ddi);
3964
3965 // Otherwise, there are cases we have to worry about where we're using a
3966 // declaration for which we must emit a definition but where we might not
3967 // find a top-level definition.
3968 // - member functions defined inline in their classes
3969 // - friend functions defined inline in some class
3970 // - special member functions with implicit definitions
3971 // If we ever change our AST traversal to walk into class methods, this
3972 // will be unnecessary.
3973 //
3974 // We also don't emit a definition for a function if it's going to be an
3975 // entry in a vtable, unless it's already marked as used.
3976 } else if (getLangOpts().CPlusPlus && d) {
3977 // Look for a declaration that's lexically in a record.
3978 for (const auto *fd = cast<FunctionDecl>(Val: d)->getMostRecentDecl(); fd;
3979 fd = fd->getPreviousDecl()) {
3980 if (isa<CXXRecordDecl>(Val: fd->getLexicalDeclContext())) {
3981 if (fd->doesThisDeclarationHaveABody()) {
3982 addDeferredDeclToEmit(GD: gd.getWithDecl(D: fd));
3983 break;
3984 }
3985 }
3986 }
3987 }
3988
3989 return funcOp;
3990}
3991
3992cir::FuncOp
3993CIRGenModule::createCIRFunction(mlir::Location loc, StringRef name,
3994 cir::FuncType funcType,
3995 const clang::FunctionDecl *funcDecl) {
3996 cir::FuncOp func;
3997 {
3998 mlir::OpBuilder::InsertionGuard guard(builder);
3999
4000 // Functions always belong at module scope, but the ambient insertion
4001 // point may be inside another op's region, e.g. a thunk body or a
4002 // global's ctor region, so it cannot be used here.
4003 builder.setInsertionPointToEnd(getModule().getBody());
4004
4005 func = cir::FuncOp::create(builder, location: loc, sym_name: name, type: funcType);
4006
4007 symbolLookupCache[func.getSymNameAttr()] = func;
4008
4009 assert(!cir::MissingFeatures::opFuncAstDeclAttr());
4010
4011 if (funcDecl && !funcDecl->hasPrototype())
4012 func.setNoProto(true);
4013
4014 assert(func.isDeclaration() && "expected empty body");
4015
4016 // A declaration gets private visibility by default, but external linkage
4017 // as the default linkage.
4018 func.setLinkageAttr(cir::GlobalLinkageKindAttr::get(
4019 context: &getMLIRContext(), value: cir::GlobalLinkageKind::ExternalLinkage));
4020 mlir::SymbolTable::setSymbolVisibility(
4021 symbol: func, vis: mlir::SymbolTable::Visibility::Private);
4022
4023 assert(!cir::MissingFeatures::opFuncExtraAttrs());
4024
4025 // Record the func_info tag, a C++ special member form or a known standard
4026 // library entity.
4027 setFuncInfoAttr(funcOp: func, funcDecl);
4028
4029 if (funcDecl && this->getLangOpts().OpenACC) {
4030 // We only have to handle this attribute, since OpenACCAnnotAttrs are
4031 // handled via the end-of-TU work.
4032 for (const auto *attr :
4033 funcDecl->specific_attrs<OpenACCRoutineDeclAttr>())
4034 emitOpenACCRoutineDecl(funcDecl, func, pragmaLoc: attr->getLocation(),
4035 clauses: attr->Clauses);
4036 }
4037 }
4038 return func;
4039}
4040
4041cir::FuncOp
4042CIRGenModule::createCIRBuiltinFunction(mlir::Location loc, StringRef name,
4043 cir::FuncType ty,
4044 const clang::FunctionDecl *fd) {
4045 cir::FuncOp fnOp = createCIRFunction(loc, name, funcType: ty, funcDecl: fd);
4046 fnOp.setBuiltin(true);
4047 return fnOp;
4048}
4049
4050static cir::CtorKind getCtorKindFromDecl(const CXXConstructorDecl *ctor) {
4051 if (ctor->isDefaultConstructor())
4052 return cir::CtorKind::Default;
4053 if (ctor->isCopyConstructor())
4054 return cir::CtorKind::Copy;
4055 if (ctor->isMoveConstructor())
4056 return cir::CtorKind::Move;
4057 return cir::CtorKind::Custom;
4058}
4059
4060static cir::AssignKind getAssignKindFromDecl(const CXXMethodDecl *method) {
4061 if (method->isCopyAssignmentOperator())
4062 return cir::AssignKind::Copy;
4063 if (method->isMoveAssignmentOperator())
4064 return cir::AssignKind::Move;
4065 llvm_unreachable("not a copy or move assignment operator");
4066}
4067
4068void CIRGenModule::setFuncInfoAttr(cir::FuncOp funcOp,
4069 const clang::FunctionDecl *funcDecl) {
4070 if (!funcDecl)
4071 return;
4072
4073 if (const auto *dtor = dyn_cast<CXXDestructorDecl>(Val: funcDecl)) {
4074 auto cxxDtor = cir::CXXDtorAttr::get(
4075 type: convertType(type: getASTContext().getCanonicalTagType(TD: dtor->getParent())),
4076 isTrivial: dtor->isTrivial());
4077 funcOp.setFuncInfoAttr(cxxDtor);
4078 return;
4079 }
4080
4081 if (const auto *ctor = dyn_cast<CXXConstructorDecl>(Val: funcDecl)) {
4082 cir::CtorKind kind = getCtorKindFromDecl(ctor);
4083 auto cxxCtor = cir::CXXCtorAttr::get(
4084 type: convertType(type: getASTContext().getCanonicalTagType(TD: ctor->getParent())),
4085 ctorKind: kind, isTrivial: ctor->isTrivial());
4086 funcOp.setFuncInfoAttr(cxxCtor);
4087 return;
4088 }
4089
4090 const auto *method = dyn_cast<CXXMethodDecl>(Val: funcDecl);
4091 if (method && (method->isCopyAssignmentOperator() ||
4092 method->isMoveAssignmentOperator())) {
4093 cir::AssignKind assignKind = getAssignKindFromDecl(method);
4094 auto cxxAssign = cir::CXXAssignAttr::get(
4095 type: convertType(type: getASTContext().getCanonicalTagType(TD: method->getParent())),
4096 assignKind, isTrivial: method->isTrivial());
4097 funcOp.setFuncInfoAttr(cxxAssign);
4098 return;
4099 }
4100
4101 // Otherwise tag a function that matches a known standard library entity. A
4102 // known entity is named by a plain identifier in std. For a member the
4103 // record decides std membership. Inline namespaces, like the versioning
4104 // namespace of libc++, count as part of std.
4105 if (!funcDecl->getIdentifier())
4106 return;
4107 bool inStdNamespace = method ? method->getParent()->isInStdNamespace()
4108 : funcDecl->isInStdNamespace();
4109 if (!inStdNamespace)
4110 return;
4111
4112 // The names and the tags come from CIRStdOps.td, and the recognizer checks
4113 // the shape of each call. Only free functions name a known entity today, so
4114 // a member like char_traits::find never shares the tag of the free std::find.
4115 std::optional<cir::KnownFuncKind> kind;
4116 if (!method) {
4117 kind = llvm::StringSwitch<std::optional<cir::KnownFuncKind>>(
4118 funcDecl->getName())
4119 .Case(S: cir::StdFindOp::getFunctionName(),
4120 Value: cir::StdFindOp::getFuncKind())
4121 .Default(Value: std::nullopt);
4122 }
4123 if (!kind)
4124 return;
4125
4126 funcOp.setFuncInfoAttr(cir::FuncIdentityAttr::get(context: &getMLIRContext(), kind: *kind));
4127}
4128
4129static void setWindowsItaniumDLLImport(CIRGenModule &cgm, bool isLocal,
4130 cir::FuncOp funcOp, StringRef name) {
4131 // In Windows Itanium environments, try to mark runtime functions
4132 // dllimport. For Mingw and MSVC, don't. We don't really know if the user
4133 // will link their standard library statically or dynamically. Marking
4134 // functions imported when they are not imported can cause linker errors
4135 // and warnings.
4136 if (!isLocal && cgm.getTarget().getTriple().isWindowsItaniumEnvironment() &&
4137 !cgm.getCodeGenOpts().LTOVisibilityPublicStd) {
4138 assert(!cir::MissingFeatures::getRuntimeFunctionDecl());
4139 assert(!cir::MissingFeatures::setDLLStorageClass());
4140 assert(!cir::MissingFeatures::opGlobalDLLImportExport());
4141 }
4142}
4143
4144cir::FuncOp CIRGenModule::createRuntimeFunction(cir::FuncType ty,
4145 StringRef name,
4146 mlir::NamedAttrList extraAttrs,
4147 bool isLocal,
4148 bool assumeConvergent) {
4149 if (assumeConvergent)
4150 errorNYI(feature: "createRuntimeFunction: assumeConvergent");
4151
4152 extraAttrs.set(name: cir::CIRDialect::getRuntimeFuncAttrName(),
4153 value: mlir::UnitAttr::get(context: &getMLIRContext()));
4154 cir::FuncOp entry = getOrCreateCIRFunction(mangledName: name, funcType: ty, gd: GlobalDecl(),
4155 /*forVtable=*/forVTable: false, extraAttrs);
4156
4157 if (entry) {
4158 // TODO(cir): set the attributes of the function.
4159 assert(!cir::MissingFeatures::setLLVMFunctionFEnvAttributes());
4160 assert(!cir::MissingFeatures::opFuncCallingConv());
4161 setWindowsItaniumDLLImport(cgm&: *this, isLocal, funcOp: entry, name);
4162 entry.setDSOLocal(true);
4163 }
4164
4165 return entry;
4166}
4167
4168mlir::SymbolTable::Visibility
4169CIRGenModule::getMLIRVisibility(cir::GlobalOp op) {
4170 // MLIR doesn't accept public symbols declarations (only
4171 // definitions).
4172 if (op.isDeclaration())
4173 return mlir::SymbolTable::Visibility::Private;
4174 return getMLIRVisibilityFromCIRLinkage(GLK: op.getLinkage());
4175}
4176
4177mlir::SymbolTable::Visibility
4178CIRGenModule::getMLIRVisibilityFromCIRLinkage(cir::GlobalLinkageKind glk) {
4179 switch (glk) {
4180 case cir::GlobalLinkageKind::InternalLinkage:
4181 case cir::GlobalLinkageKind::PrivateLinkage:
4182 return mlir::SymbolTable::Visibility::Private;
4183 case cir::GlobalLinkageKind::ExternalLinkage:
4184 case cir::GlobalLinkageKind::ExternalWeakLinkage:
4185 case cir::GlobalLinkageKind::LinkOnceODRLinkage:
4186 case cir::GlobalLinkageKind::AvailableExternallyLinkage:
4187 case cir::GlobalLinkageKind::CommonLinkage:
4188 case cir::GlobalLinkageKind::WeakAnyLinkage:
4189 case cir::GlobalLinkageKind::WeakODRLinkage:
4190 return mlir::SymbolTable::Visibility::Public;
4191 default: {
4192 llvm::errs() << "visibility not implemented for '"
4193 << stringifyGlobalLinkageKind(glk) << "'\n";
4194 assert(0 && "not implemented");
4195 }
4196 }
4197 llvm_unreachable("linkage should be handled above!");
4198}
4199
4200std::string CIRGenModule::getModuleInitializerName(clang::Module *mod) {
4201 std::string name;
4202 llvm::raw_string_ostream out(name);
4203 cast<clang::ItaniumMangleContext>(Val&: getCXXABI().getMangleContext())
4204 .mangleModuleInitializer(Module: mod, out);
4205 return name;
4206}
4207
4208void CIRGenModule::release() {
4209 emitDeferred();
4210 emitVTablesOpportunistically();
4211 applyReplacements();
4212
4213 getModule()->setAttr(name: cir::CIRDialect::getModuleLevelAsmAttrName(),
4214 value: builder.getArrayAttr(value: globalScopeAsm));
4215
4216 emitGlobalAnnotations();
4217
4218 if (std::optional<cir::FramePointerKind> kind =
4219 getFramePointerKind(kind: codeGenOpts.getFramePointer()))
4220 getModule()->setAttr(
4221 name: cir::CIRDialect::getModuleFramePointerAttrName(),
4222 value: cir::FramePointerKindAttr::get(context: &getMLIRContext(), value: *kind));
4223
4224 if (!recordLayoutEntries.empty())
4225 getModule()->setAttr(
4226 name: cir::CIRDialect::getRecordLayoutsAttrName(),
4227 value: mlir::DictionaryAttr::get(context: &getMLIRContext(), value: recordLayoutEntries));
4228
4229 if (getTriple().isAMDGPU() ||
4230 (getTriple().isSPIRV() && getTriple().getVendor() == llvm::Triple::AMD))
4231 emitAMDGPUMetadata();
4232
4233 if (getLangOpts().HIP) {
4234 // Emit a unique ID so that host and device binaries from the same
4235 // compilation unit can be associated.
4236 std::string cuidName =
4237 ("__hip_cuid_" + getASTContext().getCUIDHash()).str();
4238 auto int8Ty = cir::IntType::get(context: &getMLIRContext(), width: 8, /*isSigned=*/is_signed: false);
4239 auto loc = builder.getUnknownLoc();
4240 mlir::ptr::MemorySpaceAttrInterface addrSpace =
4241 getGlobalCIRAddressSpace(as: getGlobalVarAddressSpace(d: nullptr));
4242
4243 auto gv = createGlobalOp(loc, name: cuidName, t: int8Ty,
4244 /*isConstant=*/false, addrSpace);
4245 gv.setLinkage(cir::GlobalLinkageKind::ExternalLinkage);
4246 // Initialize with zero
4247 auto zeroAttr = cir::IntAttr::get(type: int8Ty, value: 0);
4248 gv.setInitialValueAttr(zeroAttr);
4249 // External linkage requires public visibility
4250 mlir::SymbolTable::setSymbolVisibility(
4251 symbol: gv, vis: mlir::SymbolTable::Visibility::Public);
4252
4253 addCompilerUsedGlobal(gv);
4254 }
4255
4256 if (astContext.getLangOpts().CUDA && cudaRuntime)
4257 getCUDARuntime().finalizeModule();
4258
4259 emitLLVMUsed();
4260
4261 // Precompute the mangled C++20 named-module initializer function name and
4262 // stash it on the ModuleOp so LoweringPrepare (which runs without a live
4263 // ASTContext) can read it back as an attribute. This attribute is the only
4264 // channel through which the named-module initializer reaches lowering: its
4265 // presence tells LoweringPrepare both what to call the global-init function
4266 // and that the function needs external linkage, and its absence selects the
4267 // `_GLOBAL__sub_I_` form. Lowering therefore never has to rediscover the
4268 // module from the AST.
4269 if (cxx20ModuleInits) {
4270 clang::Module *primary = astContext.getCurrentNamedModule();
4271 bool isInterfaceUnit = primary && !primary->isModuleImplementation();
4272 if (isInterfaceUnit)
4273 getModule()->setAttr(
4274 name: cir::CIRDialect::getCXXModuleInitFnNameAttrName(),
4275 value: builder.getStringAttr(bytes: getModuleInitializerName(mod: primary)));
4276
4277 // The imported modules whose initializers the global init function of
4278 // this translation unit calls first, selected as in classic CodeGen.
4279 llvm::SmallVector<mlir::Attribute> importedInits;
4280 for (clang::Module *mod : CodeGenUtils::importedModulesToInitialize(
4281 Primary: primary, Imported: importedModules.getArrayRef())) {
4282 // A module that only the interface unit's export or import lists name
4283 // takes the location of the module declaration.
4284 auto importLoc = moduleImportLocs.find(Val: mod);
4285 mlir::Location loc =
4286 getLoc(cLoc: importLoc != moduleImportLocs.end() ? importLoc->second
4287 : primary->DefinitionLoc);
4288 importedInits.push_back(Elt: cir::CXXModuleInitAttr::get(
4289 context: &getMLIRContext(),
4290 name: builder.getStringAttr(bytes: getModuleInitializerName(mod)), loc));
4291 }
4292 if (!importedInits.empty())
4293 getModule()->setAttr(name: cir::CIRDialect::getCXXModuleImportedInitsAttrName(),
4294 value: builder.getArrayAttr(value: importedInits));
4295 }
4296
4297 // Autolink metadata for the imported modules is not emitted yet.
4298 assert(!cir::MissingFeatures::emitModuleLinkOptions());
4299
4300 // Classic codegen calls `checkAliases` here to validate any alias
4301 // definitions emitted during codegen.
4302 assert(!cir::MissingFeatures::checkAliases());
4303
4304 // There's a lot of code that is not implemented yet.
4305 assert(!cir::MissingFeatures::cgmRelease());
4306}
4307
4308void CIRGenModule::emitAliasDefinition(GlobalDecl gd) {
4309 const auto *d = cast<ValueDecl>(Val: gd.getDecl());
4310 const AliasAttr *aa = d->getAttr<AliasAttr>();
4311 assert(aa && "Not an alias?");
4312
4313 StringRef mangledName = getMangledName(gd);
4314
4315 if (aa->getAliasee() == mangledName) {
4316 diags.Report(Loc: aa->getLocation(), DiagID: diag::err_cyclic_alias) << 0;
4317 return;
4318 }
4319
4320 // If there is a definition in the module, then it wins over the alias.
4321 // This is dubious, but allow it to be safe. Just ignore the alias.
4322 mlir::Operation *entry = getGlobalValue(name: mangledName);
4323 if (entry) {
4324 auto entryGV = mlir::dyn_cast<cir::CIRGlobalValueInterface>(Val: entry);
4325 if (entryGV && entryGV.isDefinition())
4326 return;
4327 }
4328
4329 // Classic codegen pushes the alias onto an `Aliases` list at this point so
4330 // that `checkAliases` can later validate the alias and recover on error.
4331 assert(!cir::MissingFeatures::checkAliases());
4332
4333 mlir::Location loc = getLoc(cRange: d->getSourceRange());
4334 bool isFunction = isa<FunctionDecl>(Val: d);
4335
4336 // Get the linkage and the type of the alias.
4337 mlir::Type declTy;
4338 cir::GlobalLinkageKind linkage;
4339 if (isFunction) {
4340 declTy = getTypes().getFunctionType(gd);
4341 linkage = getFunctionLinkage(gd);
4342 } else {
4343 declTy = getTypes().convertTypeForMem(d->getType());
4344 const auto *vd = cast<VarDecl>(Val: d);
4345 linkage = getCIRLinkageVarDefinition(vd);
4346 }
4347 //
4348 // Create the alias op.
4349 // TODO(cir): Make GlobalAlias a separate op.
4350 cir::CIRGlobalValueInterface alias =
4351 isFunction ? mlir::cast<cir::CIRGlobalValueInterface>(
4352 Val: createCIRFunction(loc, name: mangledName,
4353 funcType: mlir::cast<cir::FuncType>(Val&: declTy),
4354 funcDecl: cast<FunctionDecl>(Val: d))
4355 .getOperation())
4356 : mlir::cast<cir::CIRGlobalValueInterface>(
4357 Val: createGlobalOp(loc, name: mangledName, t: declTy).getOperation());
4358
4359 // Create the alias op. If there is an existing declaration with the same
4360 // name, erase it: any references to it via flat symbol reference will
4361 // automatically resolve to the new alias.
4362 // However, function aliases actually change its type, so we have to replace
4363 // uses of it.
4364 if (entry) {
4365 if (isFunction)
4366 replaceUsesOfNonProtoTypeWithRealFunction(
4367 old: entry, newFn: mlir::cast<cir::FuncOp>(Val: alias.getOperation()));
4368 eraseGlobalSymbol(op: entry);
4369 entry->erase();
4370 }
4371
4372 // Aliases that target weak symbols must themselves be marked weak.
4373 if (d->hasAttr<WeakAttr>() || d->hasAttr<WeakRefAttr>() ||
4374 d->isWeakImported())
4375 linkage = cir::GlobalLinkageKind::WeakAnyLinkage;
4376
4377 // Aliases are always definitions, so the MLIR visibility should match the
4378 // linkage rather than defaulting to private.
4379 mlir::SymbolTable::Visibility visibility =
4380 getMLIRVisibilityFromCIRLinkage(glk: linkage);
4381
4382 alias.setAliasee(aa->getAliasee());
4383 alias.setLinkage(linkage);
4384 mlir::SymbolTable::setSymbolVisibility(symbol: alias, vis: visibility);
4385 assert(!cir::MissingFeatures::opGlobalThreadLocal());
4386 setCommonAttributes(gd, gv: alias);
4387 assert(!cir::MissingFeatures::generateDebugInfo());
4388}
4389
4390cir::FuncOp CIRGenModule::emitAliasForGlobal(StringRef mangledName,
4391 mlir::Operation *op,
4392 GlobalDecl aliasGD,
4393 cir::FuncOp aliasee,
4394 cir::GlobalLinkageKind linkage) {
4395
4396 auto *aliasFD = dyn_cast<FunctionDecl>(Val: aliasGD.getDecl());
4397 assert(aliasFD && "expected FunctionDecl");
4398
4399 // The aliasee function type is different from the alias one, this difference
4400 // is specific to CIR because in LLVM the ptr types are already erased at this
4401 // point.
4402 const CIRGenFunctionInfo &fnInfo =
4403 getTypes().arrangeCXXStructorDeclaration(gd: aliasGD);
4404 cir::FuncType fnType = getTypes().getFunctionType(info: fnInfo);
4405
4406 cir::FuncOp alias =
4407 createCIRFunction(loc: getLoc(cRange: aliasGD.getDecl()->getSourceRange()),
4408 name: mangledName, funcType: fnType, funcDecl: aliasFD);
4409 alias.setAliasee(aliasee.getName());
4410 alias.setLinkage(linkage);
4411 // Declarations cannot have public MLIR visibility, just mark them private
4412 // but this really should have no meaning since CIR should not be using
4413 // this information to derive linkage information.
4414 mlir::SymbolTable::setSymbolVisibility(
4415 symbol: alias, vis: mlir::SymbolTable::Visibility::Private);
4416
4417 if (op) {
4418 // Any existing users of the existing function declaration will be
4419 // referencing the function by flat symbol reference (i.e. the name), so
4420 // those uses will automatically resolve to the alias now that we've
4421 // replaced the function declaration. We can safely erase the existing
4422 // function declaration.
4423 assert(cast<cir::FuncOp>(op).getFunctionType() == alias.getFunctionType() &&
4424 "declaration exists with different type");
4425 eraseGlobalSymbol(op);
4426 op->erase();
4427 } else {
4428 // Name already set by createCIRFunction
4429 }
4430
4431 // Finally, set up the alias with its proper name and attributes.
4432 setCommonAttributes(gd: aliasGD, gv: alias);
4433 return alias;
4434}
4435
4436mlir::Type CIRGenModule::convertType(QualType type) {
4437 return genTypes.convertType(type);
4438}
4439
4440bool CIRGenModule::verifyModule() const {
4441 // Verify the module after we have finished constructing it, this will
4442 // check the structural properties of the IR and invoke any specific
4443 // verifiers we have on the CIR operations.
4444 return mlir::verify(op: getModule()).succeeded();
4445}
4446
4447mlir::Attribute CIRGenModule::getAddrOfRTTIDescriptor(mlir::Location loc,
4448 QualType ty, bool forEh) {
4449 // Return a bogus pointer if RTTI is disabled, unless it's for EH.
4450 // FIXME: should we even be calling this method if RTTI is disabled
4451 // and it's not for EH?
4452 if (!shouldEmitRTTI(forEH: forEh))
4453 return builder.getConstNullPtrAttr(t: builder.getUInt8PtrTy());
4454
4455 if (forEh && ty->isObjCObjectPointerType() &&
4456 langOpts.ObjCRuntime.isGNUFamily()) {
4457 errorNYI(loc, feature: "getAddrOfRTTIDescriptor: Objc PtrType & Objc RT GUN");
4458 return {};
4459 }
4460
4461 return getCXXABI().getAddrOfRTTIDescriptor(loc, ty);
4462}
4463
4464// TODO(cir): this can be shared with LLVM codegen.
4465CharUnits CIRGenModule::computeNonVirtualBaseClassOffset(
4466 const CXXRecordDecl *derivedClass,
4467 llvm::iterator_range<CastExpr::path_const_iterator> path) {
4468 CharUnits offset = CharUnits::Zero();
4469
4470 const ASTContext &astContext = getASTContext();
4471 const CXXRecordDecl *rd = derivedClass;
4472
4473 for (const CXXBaseSpecifier *base : path) {
4474 assert(!base->isVirtual() && "Should not see virtual bases here!");
4475
4476 // Get the layout.
4477 const ASTRecordLayout &layout = astContext.getASTRecordLayout(D: rd);
4478
4479 const auto *baseDecl = base->getType()->castAsCXXRecordDecl();
4480
4481 // Add the offset.
4482 offset += layout.getBaseClassOffset(Base: baseDecl);
4483
4484 rd = baseDecl;
4485 }
4486
4487 return offset;
4488}
4489
4490DiagnosticBuilder CIRGenModule::errorNYI(SourceLocation loc,
4491 llvm::StringRef feature) {
4492 unsigned diagID = diags.getCustomDiagID(
4493 L: DiagnosticsEngine::Error, FormatString: "ClangIR code gen Not Yet Implemented: %0");
4494 return diags.Report(Loc: loc, DiagID: diagID) << feature;
4495}
4496
4497DiagnosticBuilder CIRGenModule::errorNYI(SourceRange loc,
4498 llvm::StringRef feature) {
4499 return errorNYI(loc: loc.getBegin(), feature) << loc;
4500}
4501
4502void CIRGenModule::error(SourceLocation loc, StringRef error) {
4503 unsigned diagID = getDiags().getCustomDiagID(L: DiagnosticsEngine::Error, FormatString: "%0");
4504 getDiags().Report(Loc: astContext.getFullLoc(Loc: loc), DiagID: diagID) << error;
4505}
4506
4507/// Print out an error that codegen doesn't support the specified stmt yet.
4508void CIRGenModule::errorUnsupported(const Stmt *s, llvm::StringRef type) {
4509 unsigned diagId = diags.getCustomDiagID(L: DiagnosticsEngine::Error,
4510 FormatString: "cannot compile this %0 yet");
4511 diags.Report(Loc: astContext.getFullLoc(Loc: s->getBeginLoc()), DiagID: diagId)
4512 << type << s->getSourceRange();
4513}
4514
4515/// Print out an error that codegen doesn't support the specified decl yet.
4516void CIRGenModule::errorUnsupported(const Decl *d, llvm::StringRef type) {
4517 unsigned diagId = diags.getCustomDiagID(L: DiagnosticsEngine::Error,
4518 FormatString: "cannot compile this %0 yet");
4519 diags.Report(Loc: astContext.getFullLoc(Loc: d->getLocation()), DiagID: diagId) << type;
4520}
4521
4522mlir::Operation *
4523CIRGenModule::getAddrOfGlobalTemporary(const MaterializeTemporaryExpr *mte,
4524 const Expr *init) {
4525 assert((mte->getStorageDuration() == SD_Static ||
4526 mte->getStorageDuration() == SD_Thread) &&
4527 "not a global temporary");
4528 const auto *varDecl = cast<VarDecl>(Val: mte->getExtendingDecl());
4529
4530 // Use the MaterializeTemporaryExpr's type if it has the same unqualified
4531 // base type as Init. This preserves cv-qualifiers (e.g. const from a
4532 // constexpr or const-ref binding) that skipRValueSubobjectAdjustments may
4533 // have dropped via NoOp casts, while correctly falling back to Init's type
4534 // when a real subobject adjustment changed the type (e.g. member access or
4535 // base-class cast in C++98), where E->getType() reflects the reference type,
4536 // not the actual storage type.
4537 QualType materializedType = init->getType();
4538 if (getASTContext().hasSameUnqualifiedType(T1: mte->getType(), T2: materializedType))
4539 materializedType = mte->getType();
4540
4541 CharUnits align = getASTContext().getTypeAlignInChars(T: materializedType);
4542 mlir::Location loc = getLoc(cRange: mte->getSourceRange());
4543
4544 // FIXME: If an externally-visible declaration extends multiple temporaries,
4545 // we need to give each temporary the same name in every translation unit (and
4546 // we also need to make the temporaries externally-visible).
4547 llvm::SmallString<256> name;
4548 llvm::raw_svector_ostream out(name);
4549 getCXXABI().getMangleContext().mangleReferenceTemporary(
4550 D: varDecl, ManglingNumber: mte->getManglingNumber(), out);
4551
4552 auto insertResult = materializedGlobalTemporaryMap.insert(KV: {mte, nullptr});
4553 if (!insertResult.second) {
4554 mlir::Type type = getTypes().convertTypeForMem(materializedType);
4555 // We've seen this before: either we already created it or we're in the
4556 // process of doing so.
4557 if (!insertResult.first->second) {
4558 // We recursively re-entered this function, probably during emission of
4559 // the initializer. Create a placeholder.
4560 insertResult.first->second =
4561 createGlobalOp(loc, name, t: type, /*isConstant=*/false);
4562 }
4563 return insertResult.first->second;
4564 }
4565
4566 APValue *value = nullptr;
4567 if (mte->getStorageDuration() == SD_Static && varDecl->evaluateValue()) {
4568 // If the initializer of the extending declaration is a constant
4569 // initializer, we should have a cached constant initializer for this
4570 // temporay. Note taht this m ight have a different value from the value
4571 // computed by evaluating the initializer if the surrounding constant
4572 // expression modifies the temporary.
4573 value = mte->getOrCreateValue(/*MayCreate=*/false);
4574 }
4575
4576 // Try evaluating it now, it might have a constant initializer
4577 Expr::EvalResult evalResult;
4578 if (!value && init->EvaluateAsRValue(Result&: evalResult, Ctx: getASTContext()) &&
4579 !evalResult.hasSideEffects())
4580 value = &evalResult.Val;
4581
4582 assert(!cir::MissingFeatures::addressSpace());
4583
4584 std::optional<ConstantEmitter> emitter;
4585 mlir::Attribute initialValue = nullptr;
4586 bool isConstant = false;
4587 mlir::Type type;
4588
4589 if (value) {
4590 emitter.emplace(args&: *this);
4591 initialValue = emitter->emitForInitializer(value: *value, destType: materializedType);
4592
4593 isConstant = materializedType.isConstantStorage(
4594 Ctx: getASTContext(), /*ExcludeCtor=*/value, /*ExcludeDtor=*/false);
4595
4596 type = mlir::cast<mlir::TypedAttr>(Val&: initialValue).getType();
4597 } else {
4598 // No initializer, the initialization will be provided when we initialize
4599 // the declaration which performed lifetime extension.
4600 type = getTypes().convertTypeForMem(materializedType);
4601 }
4602
4603 // Create a global variable for this lifetime-extended temporary.
4604 cir::GlobalLinkageKind linkage = getCIRLinkageVarDefinition(vd: varDecl);
4605 if (linkage == cir::GlobalLinkageKind::ExternalLinkage) {
4606 const VarDecl *initVD;
4607 if (varDecl->isStaticDataMember() && varDecl->getAnyInitializer(D&: initVD) &&
4608 isa<CXXRecordDecl>(Val: initVD->getLexicalDeclContext())) {
4609 // Temporaries defined inside a class get linkonce_odr linkage because the
4610 // calss can be defined in multiple translation units.
4611 errorNYI(loc: mte->getSourceRange(), feature: "static data member initialization");
4612 } else {
4613 // There is no need for this temporary to have external linkage if the
4614 // VarDecl has external linkage.
4615 linkage = cir::GlobalLinkageKind::InternalLinkage;
4616 }
4617 }
4618 cir::GlobalOp gv = createGlobalOp(loc, name, t: type, isConstant);
4619 gv.setInitialValueAttr(initialValue);
4620 gv.setLinkage(linkage);
4621 gv.setVisibility(getMLIRVisibilityFromCIRLinkage(glk: linkage));
4622
4623 if (emitter)
4624 emitter->finalize(gv);
4625 // Don't assign dllimport or dllexport to local linkage globals
4626 if (!gv.hasLocalLinkage()) {
4627 setGVProperties(op: gv, d: varDecl);
4628 assert(!cir::MissingFeatures::setDLLStorageClass());
4629 }
4630
4631 gv.setAlignment(align.getAsAlign().value());
4632 if (supportsCOMDAT() && gv.isWeakForLinker())
4633 gv.setSelfComdat();
4634 if (varDecl->getTLSKind())
4635 setTLSMode(op: gv, d: *varDecl, /*isExtendingDecl=*/true);
4636 mlir::Operation *cv = gv;
4637
4638 assert(!cir::MissingFeatures::addressSpace());
4639
4640 // Update the map with the new temporary. If we created a placeholder above,
4641 // erase it as well, the name will have been the same, so our symbol
4642 // references would have been correct. We still do a 'replaceAllUsesWith' in
4643 // case some sort of expression formed a reference to the placeholder
4644 // temporary.
4645 mlir::Operation *&entry = materializedGlobalTemporaryMap[mte];
4646 if (entry) {
4647 entry->replaceAllUsesWith(values&: cv);
4648 eraseGlobalSymbol(op: entry);
4649 entry->erase();
4650 }
4651 entry = cv;
4652
4653 return cv;
4654}
4655
4656cir::GlobalOp CIRGenModule::getAddrOfUnnamedGlobalConstantDecl(
4657 const UnnamedGlobalConstantDecl *gcd) {
4658 unsigned numEntries = unnamedGlobalConstantDeclMap.size();
4659 cir::GlobalOp *globalOpEntry = &unnamedGlobalConstantDeclMap[gcd];
4660
4661 if (*globalOpEntry)
4662 return *globalOpEntry;
4663
4664 ConstantEmitter emitter(*this);
4665
4666 const APValue &value = gcd->getValue();
4667 assert(!value.isAbsent());
4668 assert(!cir::MissingFeatures::addressSpace() &&
4669 "emitForInitializer should take gcd->getType().getAddressSpace()");
4670 mlir::Attribute init = emitter.emitForInitializer(value, destType: gcd->getType());
4671 auto typedInit = dyn_cast<mlir::TypedAttr>(Val&: init);
4672
4673 if (!typedInit)
4674 errorNYI(loc: gcd->getSourceRange(),
4675 feature: "getAddrOfUnnamedGlobalConstantDecl: non-typed initializer");
4676
4677 assert(!cir::MissingFeatures::addressSpace());
4678
4679 // Classic codegen always creates these with .constant, then counts on the
4680 // auto-addition of '.#'. CIR global doesn't have this, so we'll just auto-add
4681 // one if this isn't the first. We could probably choose a better name than
4682 // .constant to be unique for this type of decl, but this is consistent with
4683 // classic codegen.
4684 std::string name = numEntries == 0
4685 ? ".constant"
4686 : (Twine(".constant.") + Twine(numEntries)).str();
4687 auto globalOp = createGlobalOp(loc: builder.getUnknownLoc(), name,
4688 t: typedInit.getType(), /*is_constant=*/isConstant: true);
4689 globalOp.setLinkage(cir::GlobalLinkageKind::PrivateLinkage);
4690
4691 CharUnits alignment = getASTContext().getTypeAlignInChars(T: gcd->getType());
4692 globalOp.setAlignment(alignment.getAsAlign().value());
4693 globalOp.setUnnamedAddr(true);
4694 CIRGenModule::setInitializer(op&: globalOp, value: init);
4695
4696 emitter.finalize(gv: globalOp);
4697 *globalOpEntry = globalOp;
4698 return globalOp;
4699}
4700
4701cir::GlobalOp
4702CIRGenModule::getAddrOfTemplateParamObject(const TemplateParamObjectDecl *tpo) {
4703 StringRef name = getMangledName(gd: tpo);
4704 CharUnits alignment = getNaturalTypeAlignment(t: tpo->getType());
4705
4706 if (auto globalOp =
4707 mlir::dyn_cast_or_null<cir::GlobalOp>(Val: getGlobalValue(name)))
4708 return globalOp;
4709
4710 ConstantEmitter emitter(*this);
4711 assert(!cir::MissingFeatures::addressSpace() &&
4712 "emitForInitializer should take tpo->getType().getAddressSpace()");
4713 mlir::Attribute init =
4714 emitter.emitForInitializer(value: tpo->getValue(), destType: tpo->getType());
4715
4716 if (!init) {
4717 errorUnsupported(d: tpo, type: "template parameter object");
4718 return {};
4719 }
4720
4721 mlir::TypedAttr typedInit = cast<mlir::TypedAttr>(Val&: init);
4722
4723 cir::GlobalLinkageKind linkage =
4724 isExternallyVisible(L: tpo->getLinkageAndVisibility().getLinkage())
4725 ? cir::GlobalLinkageKind::LinkOnceODRLinkage
4726 : cir::GlobalLinkageKind::InternalLinkage;
4727
4728 assert(!cir::MissingFeatures::addressSpace());
4729 auto globalOp = createGlobalOp(loc: builder.getUnknownLoc(), name,
4730 t: typedInit.getType(), /*is_constant=*/isConstant: true);
4731 globalOp.setLinkage(linkage);
4732 globalOp.setAlignment(alignment.getAsAlign().value());
4733 if (supportsCOMDAT() && linkage == cir::GlobalLinkageKind::LinkOnceODRLinkage)
4734 globalOp.setSelfComdat();
4735
4736 CIRGenModule::setInitializer(op&: globalOp, value: init);
4737 emitter.finalize(gv: globalOp);
4738
4739 insertGlobalSymbol(op: globalOp);
4740
4741 return globalOp;
4742}
4743
4744//===----------------------------------------------------------------------===//
4745// Annotations
4746//===----------------------------------------------------------------------===//
4747
4748mlir::ArrayAttr
4749CIRGenModule::getOrCreateAnnotationArgs(const AnnotateAttr *attr) {
4750 ArrayRef<Expr *> exprs = {attr->args_begin(), attr->args_size()};
4751 // Return a null attr for no-args annotations so OptionalParameter omits
4752 // the args portion entirely from the printed IR.
4753 if (exprs.empty())
4754 return {};
4755
4756 llvm::FoldingSetNodeID id;
4757 for (Expr *e : exprs)
4758 id.Add(x: cast<clang::ConstantExpr>(Val: e)->getAPValueResult());
4759
4760 mlir::ArrayAttr &lookup = annotationArgs[id.computeHash()];
4761 if (lookup)
4762 return lookup;
4763
4764 llvm::SmallVector<mlir::Attribute> args;
4765 args.reserve(N: exprs.size());
4766 for (Expr *e : exprs) {
4767 if (auto *strE = dyn_cast<clang::StringLiteral>(Val: e->IgnoreParenCasts())) {
4768 args.push_back(Elt: builder.getStringAttr(bytes: strE->getString()));
4769 } else if (auto *intE =
4770 dyn_cast<clang::IntegerLiteral>(Val: e->IgnoreParenCasts())) {
4771 auto intTy = builder.getIntegerType(width: intE->getValue().getBitWidth());
4772 args.push_back(Elt: builder.getIntegerAttr(type: intTy, value: intE->getValue()));
4773 } else {
4774 errorNYI(loc: e->getExprLoc(), feature: "annotation argument expression");
4775 }
4776 }
4777
4778 return lookup = builder.getArrayAttr(value: args);
4779}
4780
4781cir::AnnotationAttr CIRGenModule::emitAnnotateAttr(const AnnotateAttr *aa) {
4782 mlir::StringAttr annoGV = builder.getStringAttr(bytes: aa->getAnnotation());
4783 mlir::ArrayAttr args = getOrCreateAnnotationArgs(attr: aa);
4784 return cir::AnnotationAttr::get(context: &getMLIRContext(), name: annoGV, args);
4785}
4786
4787void CIRGenModule::addGlobalAnnotations(const ValueDecl *d,
4788 mlir::Operation *gv) {
4789 assert(d->hasAttr<AnnotateAttr>() && "no annotate attribute");
4790 assert((isa<cir::GlobalOp>(gv) || isa<cir::FuncOp>(gv)) &&
4791 "annotation only on globals");
4792 llvm::SmallVector<mlir::Attribute> annotations;
4793 for (const auto *i : d->specific_attrs<AnnotateAttr>())
4794 annotations.push_back(Elt: emitAnnotateAttr(aa: i));
4795 if (auto global = dyn_cast<cir::GlobalOp>(Val: gv))
4796 global.setAnnotationsAttr(builder.getArrayAttr(value: annotations));
4797 else if (auto func = dyn_cast<cir::FuncOp>(Val: gv))
4798 func.setAnnotationsAttr(builder.getArrayAttr(value: annotations));
4799}
4800
4801void CIRGenModule::emitGlobalAnnotations() {
4802 for (const auto &[mangledName, vd] : deferredAnnotations) {
4803 mlir::Operation *gv = getGlobalValue(name: mangledName);
4804 if (gv)
4805 addGlobalAnnotations(d: vd, gv);
4806 }
4807 deferredAnnotations.clear();
4808}
4809