1//===--- CGBlocks.cpp - Emit LLVM Code for declarations ---------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit blocks.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGBlocks.h"
14#include "CGCXXABI.h"
15#include "CGDebugInfo.h"
16#include "CGObjCRuntime.h"
17#include "CGOpenCLRuntime.h"
18#include "CodeGenFunction.h"
19#include "CodeGenModule.h"
20#include "CodeGenPGO.h"
21#include "ConstantEmitter.h"
22#include "TargetInfo.h"
23#include "clang/AST/Attr.h"
24#include "clang/AST/DeclObjC.h"
25#include "clang/CodeGen/ConstantInitBuilder.h"
26#include "llvm/IR/DataLayout.h"
27#include "llvm/IR/Module.h"
28#include "llvm/Support/ScopedPrinter.h"
29#include <algorithm>
30#include <cstdio>
31
32using namespace clang;
33using namespace CodeGen;
34
35CGBlockInfo::CGBlockInfo(const BlockDecl *block, StringRef name)
36 : Name(name), CXXThisIndex(0), CanBeGlobal(false), NeedsCopyDispose(false),
37 NoEscape(false), HasCXXObject(false), UsesStret(false),
38 HasCapturedVariableLayout(false), CapturesNonExternalType(false),
39 LocalAddress(RawAddress::invalid()), StructureType(nullptr),
40 Block(block) {
41
42 // Skip asm prefix, if any. 'name' is usually taken directly from
43 // the mangled name of the enclosing function.
44 name.consume_front(Prefix: "\01");
45}
46
47// Anchor the vtable to this translation unit.
48BlockByrefHelpers::~BlockByrefHelpers() {}
49
50/// Build the given block as a global block.
51static llvm::Constant *buildGlobalBlock(CodeGenModule &CGM,
52 const CGBlockInfo &blockInfo,
53 llvm::Constant *blockFn);
54
55/// Build the helper function to copy a block.
56static llvm::Constant *buildCopyHelper(CodeGenModule &CGM,
57 const CGBlockInfo &blockInfo) {
58 return CodeGenFunction(CGM).GenerateCopyHelperFunction(blockInfo);
59}
60
61/// Build the helper function to dispose of a block.
62static llvm::Constant *buildDisposeHelper(CodeGenModule &CGM,
63 const CGBlockInfo &blockInfo) {
64 return CodeGenFunction(CGM).GenerateDestroyHelperFunction(blockInfo);
65}
66
67namespace {
68
69enum class CaptureStrKind {
70 // String for the copy helper.
71 CopyHelper,
72 // String for the dispose helper.
73 DisposeHelper,
74 // Merge the strings for the copy helper and dispose helper.
75 Merged
76};
77
78} // end anonymous namespace
79
80static std::string getBlockCaptureStr(const CGBlockInfo::Capture &Cap,
81 CaptureStrKind StrKind,
82 CharUnits BlockAlignment,
83 CodeGenModule &CGM);
84
85static std::string getBlockDescriptorName(const CGBlockInfo &BlockInfo,
86 CodeGenModule &CGM) {
87 std::string Name = "__block_descriptor_";
88 Name += llvm::to_string(Value: BlockInfo.BlockSize.getQuantity()) + "_";
89
90 if (BlockInfo.NeedsCopyDispose) {
91 if (CGM.getLangOpts().Exceptions)
92 Name += "e";
93 if (CGM.getCodeGenOpts().ObjCAutoRefCountExceptions)
94 Name += "a";
95 Name += llvm::to_string(Value: BlockInfo.BlockAlign.getQuantity()) + "_";
96
97 for (auto &Cap : BlockInfo.SortedCaptures) {
98 if (Cap.isConstantOrTrivial())
99 continue;
100
101 Name += llvm::to_string(Value: Cap.getOffset().getQuantity());
102
103 if (Cap.CopyKind == Cap.DisposeKind) {
104 // If CopyKind and DisposeKind are the same, merge the capture
105 // information.
106 assert(Cap.CopyKind != BlockCaptureEntityKind::None &&
107 "shouldn't see BlockCaptureManagedEntity that is None");
108 Name += getBlockCaptureStr(Cap, StrKind: CaptureStrKind::Merged,
109 BlockAlignment: BlockInfo.BlockAlign, CGM);
110 } else {
111 // If CopyKind and DisposeKind are not the same, which can happen when
112 // either Kind is None or the captured object is a __strong block,
113 // concatenate the copy and dispose strings.
114 Name += getBlockCaptureStr(Cap, StrKind: CaptureStrKind::CopyHelper,
115 BlockAlignment: BlockInfo.BlockAlign, CGM);
116 Name += getBlockCaptureStr(Cap, StrKind: CaptureStrKind::DisposeHelper,
117 BlockAlignment: BlockInfo.BlockAlign, CGM);
118 }
119 }
120 Name += "_";
121 }
122
123 std::string TypeAtEncoding;
124
125 if (!CGM.getCodeGenOpts().DisableBlockSignatureString) {
126 TypeAtEncoding =
127 CGM.getContext().getObjCEncodingForBlock(blockExpr: BlockInfo.getBlockExpr());
128 /// Replace occurrences of '@' with '\1'. '@' is reserved on ELF platforms
129 /// as a separator between symbol name and symbol version.
130 llvm::replace(Range&: TypeAtEncoding, OldValue: '@', NewValue: '\1');
131 }
132 Name += "e" + llvm::to_string(Value: TypeAtEncoding.size()) + "_" + TypeAtEncoding;
133 Name += "l" + CGM.getObjCRuntime().getRCBlockLayoutStr(CGM, blockInfo: BlockInfo);
134 return Name;
135}
136
137/// buildBlockDescriptor - Build the block descriptor meta-data for a block.
138/// buildBlockDescriptor is accessed from 5th field of the Block_literal
139/// meta-data and contains stationary information about the block literal.
140/// Its definition will have 4 (or optionally 6) words.
141/// \code
142/// struct Block_descriptor {
143/// unsigned long reserved;
144/// unsigned long size; // size of Block_literal metadata in bytes.
145/// void *copy_func_helper_decl; // optional copy helper.
146/// void *destroy_func_decl; // optional destructor helper.
147/// void *block_method_encoding_address; // @encode for block literal signature.
148/// void *block_layout_info; // encoding of captured block variables.
149/// };
150/// \endcode
151static llvm::Constant *buildBlockDescriptor(CodeGenModule &CGM,
152 const CGBlockInfo &blockInfo) {
153 ASTContext &C = CGM.getContext();
154
155 llvm::IntegerType *ulong =
156 cast<llvm::IntegerType>(Val: CGM.getTypes().ConvertType(T: C.UnsignedLongTy));
157 llvm::PointerType *i8p = nullptr;
158 if (CGM.getLangOpts().OpenCL)
159 i8p = llvm::PointerType::get(
160 C&: CGM.getLLVMContext(), AddressSpace: C.getTargetAddressSpace(AS: LangAS::opencl_constant));
161 else
162 i8p = CGM.VoidPtrTy;
163
164 std::string descName;
165
166 // If an equivalent block descriptor global variable exists, return it.
167 if (C.getLangOpts().ObjC &&
168 CGM.getLangOpts().getGC() == LangOptions::NonGC) {
169 descName = getBlockDescriptorName(BlockInfo: blockInfo, CGM);
170 if (llvm::GlobalValue *desc = CGM.getModule().getNamedValue(Name: descName))
171 return desc;
172 }
173
174 // If there isn't an equivalent block descriptor global variable, create a new
175 // one.
176 ConstantInitBuilder builder(CGM);
177 auto elements = builder.beginStruct();
178
179 // reserved
180 elements.addInt(intTy: ulong, value: 0);
181
182 // Size
183 // FIXME: What is the right way to say this doesn't fit? We should give
184 // a user diagnostic in that case. Better fix would be to change the
185 // API to size_t.
186 elements.addInt(intTy: ulong, value: blockInfo.BlockSize.getQuantity());
187
188 // Optional copy/dispose helpers.
189 bool hasInternalHelper = false;
190 if (blockInfo.NeedsCopyDispose) {
191 auto &Schema = CGM.getCodeGenOpts().PointerAuth.BlockHelperFunctionPointers;
192 // copy_func_helper_decl
193 llvm::Constant *copyHelper = buildCopyHelper(CGM, blockInfo);
194 elements.addSignedPointer(Pointer: copyHelper, Schema, CalleeDecl: GlobalDecl(), CalleeType: QualType());
195
196 // destroy_func_decl
197 llvm::Constant *disposeHelper = buildDisposeHelper(CGM, blockInfo);
198 elements.addSignedPointer(Pointer: disposeHelper, Schema, CalleeDecl: GlobalDecl(), CalleeType: QualType());
199
200 if (cast<llvm::Function>(Val: copyHelper->stripPointerCasts())
201 ->hasInternalLinkage() ||
202 cast<llvm::Function>(Val: disposeHelper->stripPointerCasts())
203 ->hasInternalLinkage())
204 hasInternalHelper = true;
205 }
206
207 // Signature. Mandatory ObjC-style method descriptor @encode sequence.
208 if (CGM.getCodeGenOpts().DisableBlockSignatureString) {
209 elements.addNullPointer(ptrTy: i8p);
210 } else {
211 std::string typeAtEncoding =
212 CGM.getContext().getObjCEncodingForBlock(blockExpr: blockInfo.getBlockExpr());
213 elements.add(value: CGM.GetAddrOfConstantCString(Str: typeAtEncoding).getPointer());
214 }
215
216 // GC layout.
217 if (C.getLangOpts().ObjC) {
218 if (CGM.getLangOpts().getGC() != LangOptions::NonGC)
219 elements.add(value: CGM.getObjCRuntime().BuildGCBlockLayout(CGM, blockInfo));
220 else
221 elements.add(value: CGM.getObjCRuntime().BuildRCBlockLayout(CGM, blockInfo));
222 }
223 else
224 elements.addNullPointer(ptrTy: i8p);
225
226 unsigned AddrSpace = 0;
227 if (C.getLangOpts().OpenCL)
228 AddrSpace = C.getTargetAddressSpace(AS: LangAS::opencl_constant);
229
230 llvm::GlobalValue::LinkageTypes linkage;
231 if (descName.empty()) {
232 linkage = llvm::GlobalValue::InternalLinkage;
233 descName = "__block_descriptor_tmp";
234 } else if (hasInternalHelper) {
235 // If either the copy helper or the dispose helper has internal linkage,
236 // the block descriptor must have internal linkage too.
237 linkage = llvm::GlobalValue::InternalLinkage;
238 } else {
239 linkage = llvm::GlobalValue::LinkOnceODRLinkage;
240 }
241
242 llvm::GlobalVariable *global =
243 elements.finishAndCreateGlobal(args&: descName, args: CGM.getPointerAlign(),
244 /*constant*/ args: true, args&: linkage, args&: AddrSpace);
245
246 if (linkage == llvm::GlobalValue::LinkOnceODRLinkage) {
247 if (CGM.supportsCOMDAT())
248 global->setComdat(CGM.getModule().getOrInsertComdat(Name: descName));
249 global->setVisibility(llvm::GlobalValue::HiddenVisibility);
250 global->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
251 }
252
253 return global;
254}
255
256/*
257 Purely notional variadic template describing the layout of a block.
258
259 template <class _ResultType, class... _ParamTypes, class... _CaptureTypes>
260 struct Block_literal {
261 /// Initialized to one of:
262 /// extern void *_NSConcreteStackBlock[];
263 /// extern void *_NSConcreteGlobalBlock[];
264 ///
265 /// In theory, we could start one off malloc'ed by setting
266 /// BLOCK_NEEDS_FREE, giving it a refcount of 1, and using
267 /// this isa:
268 /// extern void *_NSConcreteMallocBlock[];
269 struct objc_class *isa;
270
271 /// These are the flags (with corresponding bit number) that the
272 /// compiler is actually supposed to know about.
273 /// 23. BLOCK_IS_NOESCAPE - indicates that the block is non-escaping
274 /// 25. BLOCK_HAS_COPY_DISPOSE - indicates that the block
275 /// descriptor provides copy and dispose helper functions
276 /// 26. BLOCK_HAS_CXX_OBJ - indicates that there's a captured
277 /// object with a nontrivial destructor or copy constructor
278 /// 28. BLOCK_IS_GLOBAL - indicates that the block is allocated
279 /// as global memory
280 /// 29. BLOCK_USE_STRET - indicates that the block function
281 /// uses stret, which objc_msgSend needs to know about
282 /// 30. BLOCK_HAS_SIGNATURE - indicates that the block has an
283 /// @encoded signature string
284 /// And we're not supposed to manipulate these:
285 /// 24. BLOCK_NEEDS_FREE - indicates that the block has been moved
286 /// to malloc'ed memory
287 /// 27. BLOCK_IS_GC - indicates that the block has been moved to
288 /// to GC-allocated memory
289 /// Additionally, the bottom 16 bits are a reference count which
290 /// should be zero on the stack.
291 int flags;
292
293 /// Reserved; should be zero-initialized.
294 int reserved;
295
296 /// Function pointer generated from block literal.
297 _ResultType (*invoke)(Block_literal *, _ParamTypes...);
298
299 /// Block description metadata generated from block literal.
300 struct Block_descriptor *block_descriptor;
301
302 /// Captured values follow.
303 _CapturesTypes captures...;
304 };
305 */
306
307namespace {
308 /// A chunk of data that we actually have to capture in the block.
309 struct BlockLayoutChunk {
310 CharUnits Alignment;
311 CharUnits Size;
312 const BlockDecl::Capture *Capture; // null for 'this'
313 llvm::Type *Type;
314 QualType FieldType;
315 BlockCaptureEntityKind CopyKind, DisposeKind;
316 BlockFieldFlags CopyFlags, DisposeFlags;
317
318 BlockLayoutChunk(CharUnits align, CharUnits size,
319 const BlockDecl::Capture *capture, llvm::Type *type,
320 QualType fieldType, BlockCaptureEntityKind CopyKind,
321 BlockFieldFlags CopyFlags,
322 BlockCaptureEntityKind DisposeKind,
323 BlockFieldFlags DisposeFlags)
324 : Alignment(align), Size(size), Capture(capture), Type(type),
325 FieldType(fieldType), CopyKind(CopyKind), DisposeKind(DisposeKind),
326 CopyFlags(CopyFlags), DisposeFlags(DisposeFlags) {}
327
328 /// Tell the block info that this chunk has the given field index.
329 void setIndex(CGBlockInfo &info, unsigned index, CharUnits offset) {
330 if (!Capture) {
331 info.CXXThisIndex = index;
332 info.CXXThisOffset = offset;
333 } else {
334 info.SortedCaptures.push_back(Elt: CGBlockInfo::Capture::makeIndex(
335 index, offset, FieldType, CopyKind, CopyFlags, DisposeKind,
336 DisposeFlags, Cap: Capture));
337 }
338 }
339
340 bool isTrivial() const {
341 return CopyKind == BlockCaptureEntityKind::None &&
342 DisposeKind == BlockCaptureEntityKind::None;
343 }
344 };
345
346 /// Order by 1) all __strong together 2) next, all block together 3) next,
347 /// all byref together 4) next, all __weak together. Preserve descending
348 /// alignment in all situations.
349 bool operator<(const BlockLayoutChunk &left, const BlockLayoutChunk &right) {
350 if (left.Alignment != right.Alignment)
351 return left.Alignment > right.Alignment;
352
353 auto getPrefOrder = [](const BlockLayoutChunk &chunk) {
354 switch (chunk.CopyKind) {
355 case BlockCaptureEntityKind::ARCStrong:
356 return 0;
357 case BlockCaptureEntityKind::BlockObject:
358 switch (chunk.CopyFlags.getBitMask()) {
359 case BLOCK_FIELD_IS_OBJECT:
360 return 0;
361 case BLOCK_FIELD_IS_BLOCK:
362 return 1;
363 case BLOCK_FIELD_IS_BYREF:
364 return 2;
365 default:
366 break;
367 }
368 break;
369 case BlockCaptureEntityKind::ARCWeak:
370 return 3;
371 default:
372 break;
373 }
374 return 4;
375 };
376
377 return getPrefOrder(left) < getPrefOrder(right);
378 }
379} // end anonymous namespace
380
381static std::pair<BlockCaptureEntityKind, BlockFieldFlags>
382computeCopyInfoForBlockCapture(const BlockDecl::Capture &CI, QualType T,
383 const LangOptions &LangOpts);
384
385static std::pair<BlockCaptureEntityKind, BlockFieldFlags>
386computeDestroyInfoForBlockCapture(const BlockDecl::Capture &CI, QualType T,
387 const LangOptions &LangOpts);
388
389static void addBlockLayout(CharUnits align, CharUnits size,
390 const BlockDecl::Capture *capture, llvm::Type *type,
391 QualType fieldType,
392 SmallVectorImpl<BlockLayoutChunk> &Layout,
393 CGBlockInfo &Info, CodeGenModule &CGM) {
394 if (!capture) {
395 // 'this' capture.
396 Layout.push_back(Elt: BlockLayoutChunk(
397 align, size, capture, type, fieldType, BlockCaptureEntityKind::None,
398 BlockFieldFlags(), BlockCaptureEntityKind::None, BlockFieldFlags()));
399 return;
400 }
401
402 const LangOptions &LangOpts = CGM.getLangOpts();
403 BlockCaptureEntityKind CopyKind, DisposeKind;
404 BlockFieldFlags CopyFlags, DisposeFlags;
405
406 std::tie(args&: CopyKind, args&: CopyFlags) =
407 computeCopyInfoForBlockCapture(CI: *capture, T: fieldType, LangOpts);
408 std::tie(args&: DisposeKind, args&: DisposeFlags) =
409 computeDestroyInfoForBlockCapture(CI: *capture, T: fieldType, LangOpts);
410 Layout.push_back(Elt: BlockLayoutChunk(align, size, capture, type, fieldType,
411 CopyKind, CopyFlags, DisposeKind,
412 DisposeFlags));
413
414 if (Info.NoEscape)
415 return;
416
417 if (!Layout.back().isTrivial())
418 Info.NeedsCopyDispose = true;
419}
420
421/// Determines if the given type is safe for constant capture in C++.
422static bool isSafeForCXXConstantCapture(QualType type) {
423 const auto *record = type->getBaseElementTypeUnsafe()->getAsCXXRecordDecl();
424
425 // Only records can be unsafe.
426 if (!record)
427 return true;
428
429 // Maintain semantics for classes with non-trivial dtors or copy ctors.
430 if (!record->hasTrivialDestructor()) return false;
431 if (record->hasNonTrivialCopyConstructor()) return false;
432
433 // Otherwise, we just have to make sure there aren't any mutable
434 // fields that might have changed since initialization.
435 return !record->hasMutableFields();
436}
437
438/// It is illegal to modify a const object after initialization.
439/// Therefore, if a const object has a constant initializer, we don't
440/// actually need to keep storage for it in the block; we'll just
441/// rematerialize it at the start of the block function. This is
442/// acceptable because we make no promises about address stability of
443/// captured variables.
444static llvm::Constant *tryCaptureAsConstant(CodeGenModule &CGM,
445 CodeGenFunction *CGF,
446 const VarDecl *var) {
447 // Return if this is a function parameter. We shouldn't try to
448 // rematerialize default arguments of function parameters.
449 if (isa<ParmVarDecl>(Val: var))
450 return nullptr;
451
452 QualType type = var->getType();
453
454 // We can only do this if the variable is const.
455 if (!type.isConstQualified()) return nullptr;
456
457 // Furthermore, in C++ we have to worry about mutable fields:
458 // C++ [dcl.type.cv]p4:
459 // Except that any class member declared mutable can be
460 // modified, any attempt to modify a const object during its
461 // lifetime results in undefined behavior.
462 if (CGM.getLangOpts().CPlusPlus && !isSafeForCXXConstantCapture(type))
463 return nullptr;
464
465 // If the variable doesn't have any initializer (shouldn't this be
466 // invalid?), it's not clear what we should do. Maybe capture as
467 // zero?
468 const Expr *init = var->getInit();
469 if (!init) return nullptr;
470
471 return ConstantEmitter(CGM, CGF).tryEmitAbstractForInitializer(D: *var);
472}
473
474/// Get the low bit of a nonzero character count. This is the
475/// alignment of the nth byte if the 0th byte is universally aligned.
476static CharUnits getLowBit(CharUnits v) {
477 return CharUnits::fromQuantity(Quantity: v.getQuantity() & (~v.getQuantity() + 1));
478}
479
480static void initializeForBlockHeader(CodeGenModule &CGM, CGBlockInfo &info,
481 SmallVectorImpl<llvm::Type*> &elementTypes) {
482
483 assert(elementTypes.empty());
484 if (CGM.getLangOpts().OpenCL) {
485 // The header is basically 'struct { int; int; generic void *;
486 // custom_fields; }'. Assert that struct is packed.
487 auto GenPtrAlign = CharUnits::fromQuantity(
488 Quantity: CGM.getTarget().getPointerAlign(AddrSpace: LangAS::opencl_generic) / 8);
489 auto GenPtrSize = CharUnits::fromQuantity(
490 Quantity: CGM.getTarget().getPointerWidth(AddrSpace: LangAS::opencl_generic) / 8);
491 assert(CGM.getIntSize() <= GenPtrSize);
492 assert(CGM.getIntAlign() <= GenPtrAlign);
493 assert((2 * CGM.getIntSize()).isMultipleOf(GenPtrAlign));
494 elementTypes.push_back(Elt: CGM.IntTy); /* total size */
495 elementTypes.push_back(Elt: CGM.IntTy); /* align */
496 elementTypes.push_back(
497 Elt: CGM.getOpenCLRuntime()
498 .getGenericVoidPointerType()); /* invoke function */
499 unsigned Offset =
500 2 * CGM.getIntSize().getQuantity() + GenPtrSize.getQuantity();
501 unsigned BlockAlign = GenPtrAlign.getQuantity();
502 if (auto *Helper =
503 CGM.getTargetCodeGenInfo().getTargetOpenCLBlockHelper()) {
504 for (auto *I : Helper->getCustomFieldTypes()) /* custom fields */ {
505 // TargetOpenCLBlockHelp needs to make sure the struct is packed.
506 // If necessary, add padding fields to the custom fields.
507 unsigned Align = CGM.getDataLayout().getABITypeAlign(Ty: I).value();
508 if (BlockAlign < Align)
509 BlockAlign = Align;
510 assert(Offset % Align == 0);
511 Offset += CGM.getDataLayout().getTypeAllocSize(Ty: I);
512 elementTypes.push_back(Elt: I);
513 }
514 }
515 info.BlockAlign = CharUnits::fromQuantity(Quantity: BlockAlign);
516 info.BlockSize = CharUnits::fromQuantity(Quantity: Offset);
517 } else {
518 // The header is basically 'struct { void *; int; int; void *; void *; }'.
519 // Assert that the struct is packed.
520 assert(CGM.getIntSize() <= CGM.getPointerSize());
521 assert(CGM.getIntAlign() <= CGM.getPointerAlign());
522 assert((2 * CGM.getIntSize()).isMultipleOf(CGM.getPointerAlign()));
523 info.BlockAlign = CGM.getPointerAlign();
524 info.BlockSize = 3 * CGM.getPointerSize() + 2 * CGM.getIntSize();
525 elementTypes.push_back(Elt: CGM.VoidPtrTy);
526 elementTypes.push_back(Elt: CGM.IntTy);
527 elementTypes.push_back(Elt: CGM.IntTy);
528 elementTypes.push_back(Elt: CGM.VoidPtrTy);
529 elementTypes.push_back(Elt: CGM.getBlockDescriptorType());
530 }
531}
532
533static QualType getCaptureFieldType(const CodeGenFunction &CGF,
534 const BlockDecl::Capture &CI) {
535 const VarDecl *VD = CI.getVariable();
536
537 // If the variable is captured by an enclosing block or lambda expression,
538 // use the type of the capture field.
539 if (CGF.BlockInfo && CI.isNested())
540 return CGF.BlockInfo->getCapture(var: VD).fieldType();
541 if (auto *FD = CGF.LambdaCaptureFields.lookup(Val: VD))
542 return FD->getType();
543 // If the captured variable is a non-escaping __block variable, the field
544 // type is the reference type. If the variable is a __block variable that
545 // already has a reference type, the field type is the variable's type.
546 return VD->isNonEscapingByref() ?
547 CGF.getContext().getLValueReferenceType(T: VD->getType()) : VD->getType();
548}
549
550/// Compute the layout of the given block. Attempts to lay the block
551/// out with minimal space requirements.
552static void computeBlockInfo(CodeGenModule &CGM, CodeGenFunction *CGF,
553 CGBlockInfo &info) {
554 ASTContext &C = CGM.getContext();
555 const BlockDecl *block = info.getBlockDecl();
556
557 SmallVector<llvm::Type*, 8> elementTypes;
558 initializeForBlockHeader(CGM, info, elementTypes);
559 bool hasNonConstantCustomFields = false;
560 if (auto *OpenCLHelper =
561 CGM.getTargetCodeGenInfo().getTargetOpenCLBlockHelper())
562 hasNonConstantCustomFields =
563 !OpenCLHelper->areAllCustomFieldValuesConstant(Info: info);
564 if (!block->hasCaptures() && !hasNonConstantCustomFields) {
565 info.StructureType =
566 llvm::StructType::get(Context&: CGM.getLLVMContext(), Elements: elementTypes, isPacked: true);
567 info.CanBeGlobal = true;
568 return;
569 } else if (C.getLangOpts().ObjC &&
570 CGM.getLangOpts().getGC() == LangOptions::NonGC)
571 info.HasCapturedVariableLayout = true;
572
573 if (block->doesNotEscape())
574 info.NoEscape = true;
575
576 // Collect the layout chunks.
577 SmallVector<BlockLayoutChunk, 16> layout;
578 layout.reserve(N: block->capturesCXXThis() +
579 (block->capture_end() - block->capture_begin()));
580
581 CharUnits maxFieldAlign;
582
583 // First, 'this'.
584 if (block->capturesCXXThis()) {
585 assert(CGF && isa_and_nonnull<CXXMethodDecl>(CGF->CurFuncDecl) &&
586 "Can't capture 'this' outside a method");
587 QualType thisType = cast<CXXMethodDecl>(Val: CGF->CurFuncDecl)->getThisType();
588
589 // Theoretically, this could be in a different address space, so
590 // don't assume standard pointer size/align.
591 llvm::Type *llvmType = CGM.getTypes().ConvertType(T: thisType);
592 auto TInfo = CGM.getContext().getTypeInfoInChars(T: thisType);
593 maxFieldAlign = std::max(a: maxFieldAlign, b: TInfo.Align);
594
595 addBlockLayout(align: TInfo.Align, size: TInfo.Width, capture: nullptr, type: llvmType, fieldType: thisType,
596 Layout&: layout, Info&: info, CGM);
597 }
598
599 // Next, all the block captures.
600 for (const auto &CI : block->captures()) {
601 const VarDecl *variable = CI.getVariable();
602
603 if (CI.isEscapingByref()) {
604 // Just use void* instead of a pointer to the byref type.
605 CharUnits align = CGM.getPointerAlign();
606 maxFieldAlign = std::max(a: maxFieldAlign, b: align);
607
608 // Since a __block variable cannot be captured by lambdas, its type and
609 // the capture field type should always match.
610 assert(CGF && getCaptureFieldType(*CGF, CI) == variable->getType() &&
611 "capture type differs from the variable type");
612 addBlockLayout(align, size: CGM.getPointerSize(), capture: &CI, type: CGM.VoidPtrTy,
613 fieldType: variable->getType(), Layout&: layout, Info&: info, CGM);
614 continue;
615 }
616
617 // Otherwise, build a layout chunk with the size and alignment of
618 // the declaration.
619 if (llvm::Constant *constant = tryCaptureAsConstant(CGM, CGF, var: variable)) {
620 info.SortedCaptures.push_back(
621 Elt: CGBlockInfo::Capture::makeConstant(value: constant, Cap: &CI));
622 continue;
623 }
624
625 QualType VT = getCaptureFieldType(CGF: *CGF, CI);
626
627 if (CGM.getLangOpts().CPlusPlus)
628 if (const CXXRecordDecl *record = VT->getAsCXXRecordDecl())
629 if (CI.hasCopyExpr() || !record->hasTrivialDestructor()) {
630 info.HasCXXObject = true;
631 if (!record->isExternallyVisible())
632 info.CapturesNonExternalType = true;
633 }
634
635 CharUnits size = C.getTypeSizeInChars(T: VT);
636 CharUnits align = C.getDeclAlign(D: variable);
637
638 maxFieldAlign = std::max(a: maxFieldAlign, b: align);
639
640 llvm::Type *llvmType =
641 CGM.getTypes().ConvertTypeForMem(T: VT);
642
643 addBlockLayout(align, size, capture: &CI, type: llvmType, fieldType: VT, Layout&: layout, Info&: info, CGM);
644 }
645
646 // If that was everything, we're done here.
647 if (layout.empty()) {
648 info.StructureType =
649 llvm::StructType::get(Context&: CGM.getLLVMContext(), Elements: elementTypes, isPacked: true);
650 info.CanBeGlobal = true;
651 info.buildCaptureMap();
652 return;
653 }
654
655 // Sort the layout by alignment. We have to use a stable sort here
656 // to get reproducible results. There should probably be an
657 // llvm::array_pod_stable_sort.
658 llvm::stable_sort(Range&: layout);
659
660 // Needed for blocks layout info.
661 info.BlockHeaderForcedGapOffset = info.BlockSize;
662 info.BlockHeaderForcedGapSize = CharUnits::Zero();
663
664 CharUnits &blockSize = info.BlockSize;
665 info.BlockAlign = std::max(a: maxFieldAlign, b: info.BlockAlign);
666
667 // Assuming that the first byte in the header is maximally aligned,
668 // get the alignment of the first byte following the header.
669 CharUnits endAlign = getLowBit(v: blockSize);
670
671 // If the end of the header isn't satisfactorily aligned for the
672 // maximum thing, look for things that are okay with the header-end
673 // alignment, and keep appending them until we get something that's
674 // aligned right. This algorithm is only guaranteed optimal if
675 // that condition is satisfied at some point; otherwise we can get
676 // things like:
677 // header // next byte has alignment 4
678 // something_with_size_5; // next byte has alignment 1
679 // something_with_alignment_8;
680 // which has 7 bytes of padding, as opposed to the naive solution
681 // which might have less (?).
682 if (endAlign < maxFieldAlign) {
683 SmallVectorImpl<BlockLayoutChunk>::iterator
684 li = layout.begin() + 1, le = layout.end();
685
686 // Look for something that the header end is already
687 // satisfactorily aligned for.
688 for (; li != le && endAlign < li->Alignment; ++li)
689 ;
690
691 // If we found something that's naturally aligned for the end of
692 // the header, keep adding things...
693 if (li != le) {
694 SmallVectorImpl<BlockLayoutChunk>::iterator first = li;
695 for (; li != le; ++li) {
696 assert(endAlign >= li->Alignment);
697
698 li->setIndex(info, index: elementTypes.size(), offset: blockSize);
699 elementTypes.push_back(Elt: li->Type);
700 blockSize += li->Size;
701 endAlign = getLowBit(v: blockSize);
702
703 // ...until we get to the alignment of the maximum field.
704 if (endAlign >= maxFieldAlign) {
705 ++li;
706 break;
707 }
708 }
709 // Don't re-append everything we just appended.
710 layout.erase(CS: first, CE: li);
711 }
712 }
713
714 assert(endAlign == getLowBit(blockSize));
715
716 // At this point, we just have to add padding if the end align still
717 // isn't aligned right.
718 if (endAlign < maxFieldAlign) {
719 CharUnits newBlockSize = blockSize.alignTo(Align: maxFieldAlign);
720 CharUnits padding = newBlockSize - blockSize;
721
722 // If we haven't yet added any fields, remember that there was an
723 // initial gap; this need to go into the block layout bit map.
724 if (blockSize == info.BlockHeaderForcedGapOffset) {
725 info.BlockHeaderForcedGapSize = padding;
726 }
727
728 elementTypes.push_back(Elt: llvm::ArrayType::get(ElementType: CGM.Int8Ty,
729 NumElements: padding.getQuantity()));
730 blockSize = newBlockSize;
731 endAlign = getLowBit(v: blockSize); // might be > maxFieldAlign
732 }
733
734 assert(endAlign >= maxFieldAlign);
735 assert(endAlign == getLowBit(blockSize));
736 // Slam everything else on now. This works because they have
737 // strictly decreasing alignment and we expect that size is always a
738 // multiple of alignment.
739 for (SmallVectorImpl<BlockLayoutChunk>::iterator
740 li = layout.begin(), le = layout.end(); li != le; ++li) {
741 if (endAlign < li->Alignment) {
742 // size may not be multiple of alignment. This can only happen with
743 // an over-aligned variable. We will be adding a padding field to
744 // make the size be multiple of alignment.
745 CharUnits padding = li->Alignment - endAlign;
746 elementTypes.push_back(Elt: llvm::ArrayType::get(ElementType: CGM.Int8Ty,
747 NumElements: padding.getQuantity()));
748 blockSize += padding;
749 endAlign = getLowBit(v: blockSize);
750 }
751 assert(endAlign >= li->Alignment);
752 li->setIndex(info, index: elementTypes.size(), offset: blockSize);
753 elementTypes.push_back(Elt: li->Type);
754 blockSize += li->Size;
755 endAlign = getLowBit(v: blockSize);
756 }
757
758 info.buildCaptureMap();
759 info.StructureType =
760 llvm::StructType::get(Context&: CGM.getLLVMContext(), Elements: elementTypes, isPacked: true);
761}
762
763/// Emit a block literal expression in the current function.
764llvm::Value *CodeGenFunction::EmitBlockLiteral(const BlockExpr *blockExpr) {
765 // If the block has no captures, we won't have a pre-computed
766 // layout for it.
767 if (!blockExpr->getBlockDecl()->hasCaptures())
768 // The block literal is emitted as a global variable, and the block invoke
769 // function has to be extracted from its initializer.
770 if (llvm::Constant *Block = CGM.getAddrOfGlobalBlockIfEmitted(BE: blockExpr))
771 return Block;
772
773 CGBlockInfo blockInfo(blockExpr->getBlockDecl(), CurFn->getName());
774 computeBlockInfo(CGM, CGF: this, info&: blockInfo);
775 blockInfo.BlockExpression = blockExpr;
776 if (!blockInfo.CanBeGlobal)
777 blockInfo.LocalAddress = CreateTempAlloca(Ty: blockInfo.StructureType,
778 align: blockInfo.BlockAlign, Name: "block");
779 return EmitBlockLiteral(Info: blockInfo);
780}
781
782llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) {
783 bool IsOpenCL = CGM.getContext().getLangOpts().OpenCL;
784 llvm::PointerType *GenVoidPtrTy =
785 IsOpenCL ? CGM.getOpenCLRuntime().getGenericVoidPointerType() : VoidPtrTy;
786 LangAS GenVoidPtrAddr = IsOpenCL ? LangAS::opencl_generic : LangAS::Default;
787 auto GenVoidPtrSize = CharUnits::fromQuantity(
788 Quantity: CGM.getTarget().getPointerWidth(AddrSpace: GenVoidPtrAddr) / 8);
789 // Using the computed layout, generate the actual block function.
790 bool isLambdaConv = blockInfo.getBlockDecl()->isConversionFromLambda();
791 CodeGenFunction BlockCGF{CGM, true};
792 BlockCGF.SanOpts = SanOpts;
793 auto *InvokeFn = BlockCGF.GenerateBlockFunction(
794 GD: CurGD, Info: blockInfo, ldm: LocalDeclMap, IsLambdaConversionToBlock: isLambdaConv, BuildGlobalBlock: blockInfo.CanBeGlobal);
795 auto *blockFn = llvm::ConstantExpr::getPointerCast(C: InvokeFn, Ty: GenVoidPtrTy);
796
797 // If there is nothing to capture, we can emit this as a global block.
798 if (blockInfo.CanBeGlobal)
799 return CGM.getAddrOfGlobalBlockIfEmitted(BE: blockInfo.BlockExpression);
800
801 // Otherwise, we have to emit this as a local block.
802
803 RawAddress blockAddr = blockInfo.LocalAddress;
804 assert(blockAddr.isValid() && "block has no address!");
805
806 llvm::Constant *isa;
807 llvm::Constant *descriptor;
808 BlockFlags flags;
809 if (!IsOpenCL) {
810 // If the block is non-escaping, set field 'isa 'to NSConcreteGlobalBlock
811 // and set the BLOCK_IS_GLOBAL bit of field 'flags'. Copying a non-escaping
812 // block just returns the original block and releasing it is a no-op.
813 llvm::Constant *blockISA = blockInfo.NoEscape
814 ? CGM.getNSConcreteGlobalBlock()
815 : CGM.getNSConcreteStackBlock();
816 isa = blockISA;
817
818 // Compute the initial on-stack block flags.
819 if (!CGM.getCodeGenOpts().DisableBlockSignatureString)
820 flags = BLOCK_HAS_SIGNATURE;
821 if (blockInfo.HasCapturedVariableLayout)
822 flags |= BLOCK_HAS_EXTENDED_LAYOUT;
823 if (blockInfo.NeedsCopyDispose)
824 flags |= BLOCK_HAS_COPY_DISPOSE;
825 if (blockInfo.HasCXXObject)
826 flags |= BLOCK_HAS_CXX_OBJ;
827 if (blockInfo.UsesStret)
828 flags |= BLOCK_USE_STRET;
829 if (blockInfo.NoEscape)
830 flags |= BLOCK_IS_NOESCAPE | BLOCK_IS_GLOBAL;
831
832 // Build the block descriptor.
833 descriptor = buildBlockDescriptor(CGM, blockInfo);
834 }
835
836 auto projectField = [&](unsigned index, const Twine &name) -> Address {
837 return Builder.CreateStructGEP(Addr: blockAddr, Index: index, Name: name);
838 };
839 auto storeField = [&](llvm::Value *value, unsigned index, const Twine &name) {
840 Builder.CreateStore(Val: value, Addr: projectField(index, name));
841 };
842
843 // Initialize the block header.
844 {
845 // We assume all the header fields are densely packed.
846 unsigned index = 0;
847 CharUnits offset;
848 auto addHeaderField = [&](llvm::Value *value, CharUnits size,
849 const Twine &name) {
850 storeField(value, index, name);
851 offset += size;
852 index++;
853 };
854 auto addSignedHeaderField =
855 [&](llvm::Value *Value, const PointerAuthSchema &Schema,
856 GlobalDecl Decl, QualType Type, CharUnits Size, const Twine &Name) {
857 auto StorageAddress = projectField(index, Name);
858 if (Schema) {
859 auto AuthInfo = EmitPointerAuthInfo(
860 Schema, StorageAddress: StorageAddress.emitRawPointer(CGF&: *this), SchemaDecl: Decl, SchemaType: Type);
861 Value = EmitPointerAuthSign(Info: AuthInfo, Pointer: Value);
862 }
863 Builder.CreateStore(Val: Value, Addr: StorageAddress);
864 offset += Size;
865 index++;
866 };
867
868 if (!IsOpenCL) {
869 addSignedHeaderField(
870 isa, CGM.getCodeGenOpts().PointerAuth.ObjCIsaPointers, GlobalDecl(),
871 QualType(), getPointerSize(), "block.isa");
872 addHeaderField(llvm::ConstantInt::get(Ty: IntTy, V: flags.getBitMask()),
873 getIntSize(), "block.flags");
874 addHeaderField(llvm::ConstantInt::get(Ty: IntTy, V: 0), getIntSize(),
875 "block.reserved");
876 } else {
877 addHeaderField(
878 llvm::ConstantInt::get(Ty: IntTy, V: blockInfo.BlockSize.getQuantity()),
879 getIntSize(), "block.size");
880 addHeaderField(
881 llvm::ConstantInt::get(Ty: IntTy, V: blockInfo.BlockAlign.getQuantity()),
882 getIntSize(), "block.align");
883 }
884
885 if (!IsOpenCL) {
886 llvm::Value *blockFnPtr =
887 llvm::ConstantExpr::getBitCast(C: InvokeFn, Ty: VoidPtrTy);
888 QualType type = blockInfo.getBlockExpr()
889 ->getType()
890 ->castAs<BlockPointerType>()
891 ->getPointeeType();
892 addSignedHeaderField(
893 blockFnPtr,
894 CGM.getCodeGenOpts().PointerAuth.BlockInvocationFunctionPointers,
895 GlobalDecl(), type, getPointerSize(), "block.invoke");
896
897 addSignedHeaderField(
898 descriptor, CGM.getCodeGenOpts().PointerAuth.BlockDescriptorPointers,
899 GlobalDecl(), type, getPointerSize(), "block.descriptor");
900 } else if (auto *Helper =
901 CGM.getTargetCodeGenInfo().getTargetOpenCLBlockHelper()) {
902 addHeaderField(blockFn, GenVoidPtrSize, "block.invoke");
903 for (auto I : Helper->getCustomFieldValues(CGF&: *this, Info: blockInfo)) {
904 addHeaderField(
905 I.first,
906 CharUnits::fromQuantity(
907 Quantity: CGM.getDataLayout().getTypeAllocSize(Ty: I.first->getType())),
908 I.second);
909 }
910 } else
911 addHeaderField(blockFn, GenVoidPtrSize, "block.invoke");
912 }
913
914 // Finally, capture all the values into the block.
915 const BlockDecl *blockDecl = blockInfo.getBlockDecl();
916
917 // First, 'this'.
918 if (blockDecl->capturesCXXThis()) {
919 Address addr =
920 projectField(blockInfo.CXXThisIndex, "block.captured-this.addr");
921 Builder.CreateStore(Val: LoadCXXThis(), Addr: addr);
922 }
923
924 // Next, captured variables.
925 for (const auto &CI : blockDecl->captures()) {
926 const VarDecl *variable = CI.getVariable();
927 const CGBlockInfo::Capture &capture = blockInfo.getCapture(var: variable);
928
929 // Ignore constant captures.
930 if (capture.isConstant()) continue;
931
932 QualType type = capture.fieldType();
933
934 // This will be a [[type]]*, except that a byref entry will just be
935 // an i8**.
936 Address blockField = projectField(capture.getIndex(), "block.captured");
937
938 // Compute the address of the thing we're going to move into the
939 // block literal.
940 Address src = Address::invalid();
941
942 if (blockDecl->isConversionFromLambda()) {
943 // The lambda capture in a lambda's conversion-to-block-pointer is
944 // special; we'll simply emit it directly.
945 src = Address::invalid();
946 } else if (CI.isEscapingByref()) {
947 if (BlockInfo && CI.isNested()) {
948 // We need to use the capture from the enclosing block.
949 const CGBlockInfo::Capture &enclosingCapture =
950 BlockInfo->getCapture(var: variable);
951
952 // This is a [[type]]*, except that a byref entry will just be an i8**.
953 src = Builder.CreateStructGEP(Addr: LoadBlockStruct(),
954 Index: enclosingCapture.getIndex(),
955 Name: "block.capture.addr");
956 } else {
957 auto I = LocalDeclMap.find(Val: variable);
958 assert(I != LocalDeclMap.end());
959 src = I->second;
960 }
961 } else {
962 DeclRefExpr declRef(getContext(), const_cast<VarDecl *>(variable),
963 /*RefersToEnclosingVariableOrCapture*/ CI.isNested(),
964 type.getNonReferenceType(), VK_LValue,
965 SourceLocation());
966 src = EmitDeclRefLValue(E: &declRef).getAddress();
967 };
968
969 // For byrefs, we just write the pointer to the byref struct into
970 // the block field. There's no need to chase the forwarding
971 // pointer at this point, since we're building something that will
972 // live a shorter life than the stack byref anyway.
973 if (CI.isEscapingByref()) {
974 // Get a void* that points to the byref struct.
975 llvm::Value *byrefPointer;
976 if (CI.isNested())
977 byrefPointer = Builder.CreateLoad(Addr: src, Name: "byref.capture");
978 else
979 byrefPointer = src.emitRawPointer(CGF&: *this);
980
981 // Write that void* into the capture field.
982 Builder.CreateStore(Val: byrefPointer, Addr: blockField);
983
984 // If we have a copy constructor, evaluate that into the block field.
985 } else if (const Expr *copyExpr = CI.getCopyExpr()) {
986 if (blockDecl->isConversionFromLambda()) {
987 // If we have a lambda conversion, emit the expression
988 // directly into the block instead.
989 AggValueSlot Slot =
990 AggValueSlot::forAddr(addr: blockField, quals: Qualifiers(),
991 isDestructed: AggValueSlot::IsDestructed,
992 needsGC: AggValueSlot::DoesNotNeedGCBarriers,
993 isAliased: AggValueSlot::IsNotAliased,
994 mayOverlap: AggValueSlot::DoesNotOverlap);
995 EmitAggExpr(E: copyExpr, AS: Slot);
996 } else {
997 EmitSynthesizedCXXCopyCtor(Dest: blockField, Src: src, Exp: copyExpr);
998 }
999
1000 // If it's a reference variable, copy the reference into the block field.
1001 } else if (type->getAs<ReferenceType>()) {
1002 Builder.CreateStore(Val: src.emitRawPointer(CGF&: *this), Addr: blockField);
1003
1004 // If type is const-qualified, copy the value into the block field.
1005 } else if (type.isConstQualified() &&
1006 type.getObjCLifetime() == Qualifiers::OCL_Strong &&
1007 CGM.getCodeGenOpts().OptimizationLevel != 0) {
1008 llvm::Value *value = Builder.CreateLoad(Addr: src, Name: "captured");
1009 Builder.CreateStore(Val: value, Addr: blockField);
1010
1011 // If this is an ARC __strong block-pointer variable, don't do a
1012 // block copy.
1013 //
1014 // TODO: this can be generalized into the normal initialization logic:
1015 // we should never need to do a block-copy when initializing a local
1016 // variable, because the local variable's lifetime should be strictly
1017 // contained within the stack block's.
1018 } else if (type.getObjCLifetime() == Qualifiers::OCL_Strong &&
1019 type->isBlockPointerType()) {
1020 // Load the block and do a simple retain.
1021 llvm::Value *value = Builder.CreateLoad(Addr: src, Name: "block.captured_block");
1022 value = EmitARCRetainNonBlock(value);
1023
1024 // Do a primitive store to the block field.
1025 Builder.CreateStore(Val: value, Addr: blockField);
1026
1027 // Otherwise, fake up a POD copy into the block field.
1028 } else {
1029 // Fake up a new variable so that EmitScalarInit doesn't think
1030 // we're referring to the variable in its own initializer.
1031 auto *BlockFieldPseudoVar = ImplicitParamDecl::Create(
1032 C&: getContext(), T: type, ParamKind: ImplicitParamKind::Other);
1033
1034 // We use one of these or the other depending on whether the
1035 // reference is nested.
1036 DeclRefExpr declRef(getContext(), const_cast<VarDecl *>(variable),
1037 /*RefersToEnclosingVariableOrCapture*/ CI.isNested(),
1038 type, VK_LValue, SourceLocation());
1039
1040 ImplicitCastExpr l2r(ImplicitCastExpr::OnStack, type, CK_LValueToRValue,
1041 &declRef, VK_PRValue, FPOptionsOverride());
1042 // FIXME: Pass a specific location for the expr init so that the store is
1043 // attributed to a reasonable location - otherwise it may be attributed to
1044 // locations of subexpressions in the initialization.
1045 EmitExprAsInit(init: &l2r, D: BlockFieldPseudoVar,
1046 lvalue: MakeAddrLValue(Addr: blockField, T: type, Source: AlignmentSource::Decl),
1047 /*captured by init*/ capturedByInit: false);
1048 }
1049
1050 // Push a cleanup for the capture if necessary.
1051 if (!blockInfo.NoEscape && !blockInfo.NeedsCopyDispose)
1052 continue;
1053
1054 // Ignore __block captures; there's nothing special in the on-stack block
1055 // that we need to do for them.
1056 if (CI.isByRef())
1057 continue;
1058
1059 // Ignore objects that aren't destructed.
1060 QualType::DestructionKind dtorKind = type.isDestructedType();
1061 if (dtorKind == QualType::DK_none)
1062 continue;
1063
1064 CodeGenFunction::Destroyer *destroyer;
1065
1066 // Block captures count as local values and have imprecise semantics.
1067 // They also can't be arrays, so need to worry about that.
1068 //
1069 // For const-qualified captures, emit clang.arc.use to ensure the captured
1070 // object doesn't get released while we are still depending on its validity
1071 // within the block.
1072 if (type.isConstQualified() &&
1073 type.getObjCLifetime() == Qualifiers::OCL_Strong &&
1074 CGM.getCodeGenOpts().OptimizationLevel != 0) {
1075 assert(CGM.getLangOpts().ObjCAutoRefCount &&
1076 "expected ObjC ARC to be enabled");
1077 destroyer = emitARCIntrinsicUse;
1078 } else if (dtorKind == QualType::DK_objc_strong_lifetime) {
1079 destroyer = destroyARCStrongImprecise;
1080 } else {
1081 destroyer = getDestroyer(destructionKind: dtorKind);
1082 }
1083
1084 CleanupKind cleanupKind = NormalCleanup;
1085 bool useArrayEHCleanup = needsEHCleanup(kind: dtorKind);
1086 if (useArrayEHCleanup)
1087 cleanupKind = NormalAndEHCleanup;
1088
1089 // Extend the lifetime of the capture to the end of the scope enclosing the
1090 // block expression except when the block decl is in the list of RetExpr's
1091 // cleanup objects, in which case its lifetime ends after the full
1092 // expression.
1093 auto IsBlockDeclInRetExpr = [&]() {
1094 auto *EWC = llvm::dyn_cast_or_null<ExprWithCleanups>(Val: RetExpr);
1095 if (EWC)
1096 for (auto &C : EWC->getObjects())
1097 if (auto *BD = C.dyn_cast<BlockDecl *>())
1098 if (BD == blockDecl)
1099 return true;
1100 return false;
1101 };
1102
1103 if (IsBlockDeclInRetExpr())
1104 pushDestroy(kind: cleanupKind, addr: blockField, type, destroyer, useEHCleanupForArray: useArrayEHCleanup);
1105 else
1106 pushLifetimeExtendedDestroy(kind: cleanupKind, addr: blockField, type, destroyer,
1107 useEHCleanupForArray: useArrayEHCleanup);
1108 }
1109
1110 // Cast to the converted block-pointer type, which happens (somewhat
1111 // unfortunately) to be a pointer to function type.
1112 llvm::Value *result = Builder.CreatePointerCast(
1113 V: blockAddr.getPointer(), DestTy: ConvertType(T: blockInfo.getBlockExpr()->getType()));
1114
1115 if (IsOpenCL) {
1116 CGM.getOpenCLRuntime().recordBlockInfo(E: blockInfo.BlockExpression, InvokeF: InvokeFn,
1117 Block: result, BlockTy: blockInfo.StructureType);
1118 }
1119
1120 return result;
1121}
1122
1123
1124llvm::Type *CodeGenModule::getBlockDescriptorType() {
1125 if (BlockDescriptorType)
1126 return BlockDescriptorType;
1127
1128 unsigned AddrSpace = 0;
1129 if (getLangOpts().OpenCL)
1130 AddrSpace = getContext().getTargetAddressSpace(AS: LangAS::opencl_constant);
1131 BlockDescriptorType = llvm::PointerType::get(C&: getLLVMContext(), AddressSpace: AddrSpace);
1132 return BlockDescriptorType;
1133}
1134
1135llvm::Type *CodeGenModule::getGenericBlockLiteralType() {
1136 if (GenericBlockLiteralType)
1137 return GenericBlockLiteralType;
1138
1139 llvm::Type *BlockDescPtrTy = getBlockDescriptorType();
1140
1141 if (getLangOpts().OpenCL) {
1142 // struct __opencl_block_literal_generic {
1143 // int __size;
1144 // int __align;
1145 // __generic void *__invoke;
1146 // /* custom fields */
1147 // };
1148 SmallVector<llvm::Type *, 8> StructFields(
1149 {IntTy, IntTy, getOpenCLRuntime().getGenericVoidPointerType()});
1150 if (auto *Helper = getTargetCodeGenInfo().getTargetOpenCLBlockHelper()) {
1151 llvm::append_range(C&: StructFields, R: Helper->getCustomFieldTypes());
1152 }
1153 GenericBlockLiteralType = llvm::StructType::create(
1154 Elements: StructFields, Name: "struct.__opencl_block_literal_generic");
1155 } else {
1156 // struct __block_literal_generic {
1157 // void *__isa;
1158 // int __flags;
1159 // int __reserved;
1160 // void (*__invoke)(void *);
1161 // struct __block_descriptor *__descriptor;
1162 // };
1163 GenericBlockLiteralType =
1164 llvm::StructType::create(Name: "struct.__block_literal_generic", elt1: VoidPtrTy,
1165 elts: IntTy, elts: IntTy, elts: VoidPtrTy, elts: BlockDescPtrTy);
1166 }
1167
1168 return GenericBlockLiteralType;
1169}
1170
1171RValue CodeGenFunction::EmitBlockCallExpr(const CallExpr *E,
1172 ReturnValueSlot ReturnValue,
1173 llvm::CallBase **CallOrInvoke) {
1174 const auto *BPT = E->getCallee()->getType()->castAs<BlockPointerType>();
1175 llvm::Value *BlockPtr = EmitScalarExpr(E: E->getCallee());
1176 llvm::Type *GenBlockTy = CGM.getGenericBlockLiteralType();
1177 llvm::Value *Func = nullptr;
1178 QualType FnType = BPT->getPointeeType();
1179 ASTContext &Ctx = getContext();
1180 CallArgList Args;
1181
1182 llvm::Value *FuncPtr = nullptr;
1183
1184 if (getLangOpts().OpenCL) {
1185 // For OpenCL, BlockPtr is already casted to generic block literal.
1186
1187 // First argument of a block call is a generic block literal casted to
1188 // generic void pointer, i.e. i8 addrspace(4)*
1189 llvm::Type *GenericVoidPtrTy =
1190 CGM.getOpenCLRuntime().getGenericVoidPointerType();
1191 llvm::Value *BlockDescriptor = Builder.CreatePointerCast(
1192 V: BlockPtr, DestTy: GenericVoidPtrTy);
1193 QualType VoidPtrQualTy = Ctx.getPointerType(
1194 T: Ctx.getAddrSpaceQualType(T: Ctx.VoidTy, AddressSpace: LangAS::opencl_generic));
1195 Args.add(rvalue: RValue::get(V: BlockDescriptor), type: VoidPtrQualTy);
1196 // And the rest of the arguments.
1197 EmitCallArgs(Args, Prototype: FnType->getAs<FunctionProtoType>(), ArgRange: E->arguments());
1198
1199 // We *can* call the block directly unless it is a function argument.
1200 if (!isa<ParmVarDecl>(Val: E->getCalleeDecl()))
1201 Func = CGM.getOpenCLRuntime().getInvokeFunction(E: E->getCallee());
1202 else {
1203 FuncPtr = Builder.CreateStructGEP(Ty: GenBlockTy, Ptr: BlockPtr, Idx: 2);
1204 Func = Builder.CreateAlignedLoad(Ty: GenericVoidPtrTy, Addr: FuncPtr,
1205 Align: getPointerAlign());
1206 }
1207 } else {
1208 // Bitcast the block literal to a generic block literal.
1209 BlockPtr =
1210 Builder.CreatePointerCast(V: BlockPtr, DestTy: DefaultPtrTy, Name: "block.literal");
1211 // Get pointer to the block invoke function
1212 FuncPtr = Builder.CreateStructGEP(Ty: GenBlockTy, Ptr: BlockPtr, Idx: 3);
1213
1214 // First argument is a block literal casted to a void pointer
1215 BlockPtr = Builder.CreatePointerCast(V: BlockPtr, DestTy: VoidPtrTy);
1216 Args.add(rvalue: RValue::get(V: BlockPtr), type: Ctx.VoidPtrTy);
1217 // And the rest of the arguments.
1218 EmitCallArgs(Args, Prototype: FnType->getAs<FunctionProtoType>(), ArgRange: E->arguments());
1219
1220 // Load the function.
1221 Func = Builder.CreateAlignedLoad(Ty: VoidPtrTy, Addr: FuncPtr, Align: getPointerAlign());
1222 }
1223
1224 const FunctionType *FuncTy = FnType->castAs<FunctionType>();
1225 const CGFunctionInfo &FnInfo =
1226 CGM.getTypes().arrangeBlockFunctionCall(args: Args, type: FuncTy);
1227
1228 // Prepare the callee.
1229 CGPointerAuthInfo PointerAuth;
1230 if (auto &AuthSchema =
1231 CGM.getCodeGenOpts().PointerAuth.BlockInvocationFunctionPointers) {
1232 assert(FuncPtr != nullptr && "Missing function pointer for AuthInfo");
1233 PointerAuth =
1234 EmitPointerAuthInfo(Schema: AuthSchema, StorageAddress: FuncPtr, SchemaDecl: GlobalDecl(), SchemaType: FnType);
1235 }
1236
1237 CGCallee Callee(CGCalleeInfo(), Func, PointerAuth);
1238
1239 // And call the block.
1240 return EmitCall(CallInfo: FnInfo, Callee, ReturnValue, Args, CallOrInvoke);
1241}
1242
1243Address CodeGenFunction::GetAddrOfBlockDecl(const VarDecl *variable) {
1244 assert(BlockInfo && "evaluating block ref without block information?");
1245 const CGBlockInfo::Capture &capture = BlockInfo->getCapture(var: variable);
1246
1247 // Handle constant captures.
1248 if (capture.isConstant()) return LocalDeclMap.find(Val: variable)->second;
1249
1250 Address addr = Builder.CreateStructGEP(Addr: LoadBlockStruct(), Index: capture.getIndex(),
1251 Name: "block.capture.addr");
1252
1253 if (variable->isEscapingByref()) {
1254 // addr should be a void** right now. Load, then cast the result
1255 // to byref*.
1256
1257 auto &byrefInfo = getBlockByrefInfo(var: variable);
1258 addr = Address(Builder.CreateLoad(Addr: addr), byrefInfo.Type,
1259 byrefInfo.ByrefAlignment);
1260
1261 addr = emitBlockByrefAddress(baseAddr: addr, info: byrefInfo, /*follow*/ followForward: true,
1262 name: variable->getName());
1263 }
1264
1265 assert((!variable->isNonEscapingByref() ||
1266 capture.fieldType()->isReferenceType()) &&
1267 "the capture field of a non-escaping variable should have a "
1268 "reference type");
1269 if (capture.fieldType()->isReferenceType())
1270 addr = EmitLoadOfReference(RefLVal: MakeAddrLValue(Addr: addr, T: capture.fieldType()));
1271
1272 return addr;
1273}
1274
1275void CodeGenModule::setAddrOfGlobalBlock(const BlockExpr *BE,
1276 llvm::Constant *Addr) {
1277 bool Ok = EmittedGlobalBlocks.insert(KV: std::make_pair(x&: BE, y&: Addr)).second;
1278 (void)Ok;
1279 assert(Ok && "Trying to replace an already-existing global block!");
1280}
1281
1282llvm::Constant *
1283CodeGenModule::GetAddrOfGlobalBlock(const BlockExpr *BE,
1284 StringRef Name) {
1285 if (llvm::Constant *Block = getAddrOfGlobalBlockIfEmitted(BE))
1286 return Block;
1287
1288 CGBlockInfo blockInfo(BE->getBlockDecl(), Name);
1289 blockInfo.BlockExpression = BE;
1290
1291 // Compute information about the layout, etc., of this block.
1292 computeBlockInfo(CGM&: *this, CGF: nullptr, info&: blockInfo);
1293
1294 // Using that metadata, generate the actual block function.
1295 {
1296 CodeGenFunction::DeclMapTy LocalDeclMap;
1297 CodeGenFunction(*this).GenerateBlockFunction(
1298 GD: GlobalDecl(), Info: blockInfo, ldm: LocalDeclMap,
1299 /*IsLambdaConversionToBlock*/ false, /*BuildGlobalBlock*/ true);
1300 }
1301
1302 return getAddrOfGlobalBlockIfEmitted(BE);
1303}
1304
1305static llvm::Constant *buildGlobalBlock(CodeGenModule &CGM,
1306 const CGBlockInfo &blockInfo,
1307 llvm::Constant *blockFn) {
1308 assert(blockInfo.CanBeGlobal);
1309 // Callers should detect this case on their own: calling this function
1310 // generally requires computing layout information, which is a waste of time
1311 // if we've already emitted this block.
1312 assert(!CGM.getAddrOfGlobalBlockIfEmitted(blockInfo.BlockExpression) &&
1313 "Refusing to re-emit a global block.");
1314
1315 // Generate the constants for the block literal initializer.
1316 ConstantInitBuilder builder(CGM);
1317 auto fields = builder.beginStruct();
1318
1319 bool IsOpenCL = CGM.getLangOpts().OpenCL;
1320 bool IsWindows = CGM.getTarget().getTriple().isOSWindows();
1321 auto &CGOPointerAuth = CGM.getCodeGenOpts().PointerAuth;
1322 if (!IsOpenCL) {
1323 // isa
1324 if (IsWindows)
1325 fields.addNullPointer(ptrTy: CGM.Int8PtrPtrTy);
1326 else
1327 fields.addSignedPointer(Pointer: CGM.getNSConcreteGlobalBlock(),
1328 Schema: CGOPointerAuth.ObjCIsaPointers, CalleeDecl: GlobalDecl(),
1329 CalleeType: QualType());
1330
1331 // __flags
1332 BlockFlags flags = BLOCK_IS_GLOBAL;
1333 if (!CGM.getCodeGenOpts().DisableBlockSignatureString)
1334 flags |= BLOCK_HAS_SIGNATURE;
1335 if (blockInfo.UsesStret)
1336 flags |= BLOCK_USE_STRET;
1337
1338 fields.addInt(intTy: CGM.IntTy, value: flags.getBitMask());
1339
1340 // Reserved
1341 fields.addInt(intTy: CGM.IntTy, value: 0);
1342 } else {
1343 fields.addInt(intTy: CGM.IntTy, value: blockInfo.BlockSize.getQuantity());
1344 fields.addInt(intTy: CGM.IntTy, value: blockInfo.BlockAlign.getQuantity());
1345 }
1346
1347 // Function
1348 if (auto &Schema = CGOPointerAuth.BlockInvocationFunctionPointers) {
1349 QualType FnType = blockInfo.getBlockExpr()
1350 ->getType()
1351 ->castAs<BlockPointerType>()
1352 ->getPointeeType();
1353 fields.addSignedPointer(Pointer: blockFn, Schema, CalleeDecl: GlobalDecl(), CalleeType: FnType);
1354 } else
1355 fields.add(value: blockFn);
1356
1357 if (!IsOpenCL) {
1358 // Descriptor
1359 llvm::Constant *Descriptor = buildBlockDescriptor(CGM, blockInfo);
1360 fields.addSignedPointer(Pointer: Descriptor, Schema: CGOPointerAuth.BlockDescriptorPointers,
1361 CalleeDecl: GlobalDecl(), CalleeType: QualType());
1362 } else if (auto *Helper =
1363 CGM.getTargetCodeGenInfo().getTargetOpenCLBlockHelper()) {
1364 for (auto *I : Helper->getCustomFieldValues(CGM, Info: blockInfo)) {
1365 fields.add(value: I);
1366 }
1367 }
1368
1369 unsigned AddrSpace = 0;
1370 if (CGM.getContext().getLangOpts().OpenCL)
1371 AddrSpace = CGM.getContext().getTargetAddressSpace(AS: LangAS::opencl_global);
1372
1373 llvm::GlobalVariable *literal = fields.finishAndCreateGlobal(
1374 args: "__block_literal_global", args: blockInfo.BlockAlign,
1375 /*constant*/ args: !IsWindows, args: llvm::GlobalVariable::InternalLinkage, args&: AddrSpace);
1376
1377 literal->addAttribute(Kind: "objc_arc_inert");
1378
1379 // Windows does not allow globals to be initialised to point to globals in
1380 // different DLLs. Any such variables must run code to initialise them.
1381 if (IsWindows) {
1382 auto *Init = llvm::Function::Create(Ty: llvm::FunctionType::get(Result: CGM.VoidTy,
1383 isVarArg: {}), Linkage: llvm::GlobalValue::InternalLinkage, N: ".block_isa_init",
1384 M: &CGM.getModule());
1385 llvm::IRBuilder<> b(llvm::BasicBlock::Create(Context&: CGM.getLLVMContext(), Name: "entry",
1386 Parent: Init));
1387 b.CreateAlignedStore(Val: CGM.getNSConcreteGlobalBlock(),
1388 Ptr: b.CreateStructGEP(Ty: literal->getValueType(), Ptr: literal, Idx: 0),
1389 Align: CGM.getPointerAlign().getAsAlign());
1390 b.CreateRetVoid();
1391 // We can't use the normal LLVM global initialisation array, because we
1392 // need to specify that this runs early in library initialisation.
1393 auto *InitVar = new llvm::GlobalVariable(CGM.getModule(), Init->getType(),
1394 /*isConstant*/true, llvm::GlobalValue::InternalLinkage,
1395 Init, ".block_isa_init_ptr");
1396 InitVar->setSection(".CRT$XCLa");
1397 CGM.addUsedGlobal(GV: InitVar);
1398 }
1399
1400 // Return a constant of the appropriately-casted type.
1401 llvm::Type *RequiredType =
1402 CGM.getTypes().ConvertType(T: blockInfo.getBlockExpr()->getType());
1403 llvm::Constant *Result =
1404 llvm::ConstantExpr::getPointerCast(C: literal, Ty: RequiredType);
1405 CGM.setAddrOfGlobalBlock(BE: blockInfo.BlockExpression, Addr: Result);
1406 if (CGM.getContext().getLangOpts().OpenCL)
1407 CGM.getOpenCLRuntime().recordBlockInfo(
1408 E: blockInfo.BlockExpression,
1409 InvokeF: cast<llvm::Function>(Val: blockFn->stripPointerCasts()), Block: Result,
1410 BlockTy: literal->getValueType());
1411 return Result;
1412}
1413
1414void CodeGenFunction::setBlockContextParameter(const ImplicitParamDecl *D,
1415 unsigned argNum,
1416 llvm::Value *arg) {
1417 assert(BlockInfo && "not emitting prologue of block invocation function?!");
1418
1419 // Allocate a stack slot like for any local variable to guarantee optimal
1420 // debug info at -O0. The mem2reg pass will eliminate it when optimizing.
1421 RawAddress alloc =
1422 CreateMemTempWithoutCast(T: D->getType(), Name: D->getName() + ".addr");
1423 Builder.CreateStore(Val: arg, Addr: alloc);
1424 if (CGDebugInfo *DI = getDebugInfo()) {
1425 if (CGM.getCodeGenOpts().hasReducedDebugInfo()) {
1426 DI->setLocation(D->getLocation());
1427 DI->EmitDeclareOfBlockLiteralArgVariable(
1428 block: *BlockInfo, Name: D->getName(), ArgNo: argNum,
1429 LocalAddr: cast<llvm::AllocaInst>(Val: alloc.getPointer()->stripPointerCasts()),
1430 Builder);
1431 }
1432 }
1433
1434 SourceLocation StartLoc = BlockInfo->getBlockExpr()->getBody()->getBeginLoc();
1435 ApplyDebugLocation Scope(*this, StartLoc);
1436
1437 // Instead of messing around with LocalDeclMap, just set the value
1438 // directly as BlockPointer.
1439 BlockPointer = Builder.CreatePointerCast(
1440 V: arg,
1441 DestTy: llvm::PointerType::get(
1442 C&: getLLVMContext(),
1443 AddressSpace: getContext().getLangOpts().OpenCL
1444 ? getContext().getTargetAddressSpace(AS: LangAS::opencl_generic)
1445 : 0),
1446 Name: "block");
1447}
1448
1449Address CodeGenFunction::LoadBlockStruct() {
1450 assert(BlockInfo && "not in a block invocation function!");
1451 assert(BlockPointer && "no block pointer set!");
1452 return Address(BlockPointer, BlockInfo->StructureType, BlockInfo->BlockAlign);
1453}
1454
1455llvm::Function *CodeGenFunction::GenerateBlockFunction(
1456 GlobalDecl GD, const CGBlockInfo &blockInfo, const DeclMapTy &ldm,
1457 bool IsLambdaConversionToBlock, bool BuildGlobalBlock) {
1458 const BlockDecl *blockDecl = blockInfo.getBlockDecl();
1459
1460 CurGD = GD;
1461
1462 CurEHLocation = blockInfo.getBlockExpr()->getEndLoc();
1463
1464 BlockInfo = &blockInfo;
1465
1466 // Arrange for local static and local extern declarations to appear
1467 // to be local to this function as well, in case they're directly
1468 // referenced in a block.
1469 for (const auto &KV : ldm) {
1470 const auto *var = dyn_cast<VarDecl>(Val: KV.first);
1471 if (var && !var->hasLocalStorage())
1472 setAddrOfLocalVar(VD: var, Addr: KV.second);
1473 }
1474
1475 // Begin building the function declaration.
1476
1477 // Build the argument list.
1478 FunctionArgList args;
1479
1480 // The first argument is the block pointer. Just take it as a void*
1481 // and cast it later.
1482 QualType selfTy = getContext().VoidPtrTy;
1483
1484 // For OpenCL passed block pointer can be private AS local variable or
1485 // global AS program scope variable (for the case with and without captures).
1486 // Generic AS is used therefore to be able to accommodate both private and
1487 // generic AS in one implementation.
1488 if (getLangOpts().OpenCL)
1489 selfTy = getContext().getPointerType(T: getContext().getAddrSpaceQualType(
1490 T: getContext().VoidTy, AddressSpace: LangAS::opencl_generic));
1491
1492 const IdentifierInfo *II = &CGM.getContext().Idents.get(Name: ".block_descriptor");
1493
1494 auto *SelfDecl = ImplicitParamDecl::Create(
1495 C&: getContext(), DC: const_cast<BlockDecl *>(blockDecl), IdLoc: SourceLocation(), Id: II,
1496 T: selfTy, ParamKind: ImplicitParamKind::ObjCSelf);
1497 args.push_back(Elt: SelfDecl);
1498
1499 // Now add the rest of the parameters.
1500 args.append(in_start: blockDecl->param_begin(), in_end: blockDecl->param_end());
1501
1502 // Create the function declaration.
1503 const FunctionProtoType *fnType = blockInfo.getBlockExpr()->getFunctionType();
1504 const CGFunctionInfo &fnInfo =
1505 CGM.getTypes().arrangeBlockFunctionDeclaration(type: fnType, args);
1506 if (CGM.ReturnSlotInterferesWithArgs(FI: fnInfo))
1507 blockInfo.UsesStret = true;
1508
1509 llvm::FunctionType *fnLLVMType = CGM.getTypes().GetFunctionType(Info: fnInfo);
1510
1511 StringRef name = CGM.getBlockMangledName(GD, BD: blockDecl);
1512 llvm::Function *fn = llvm::Function::Create(
1513 Ty: fnLLVMType, Linkage: llvm::GlobalValue::InternalLinkage, N: name, M: &CGM.getModule());
1514 CGM.SetInternalFunctionAttributes(GD: blockDecl, F: fn, FI: fnInfo);
1515
1516 if (BuildGlobalBlock) {
1517 auto GenVoidPtrTy = getContext().getLangOpts().OpenCL
1518 ? CGM.getOpenCLRuntime().getGenericVoidPointerType()
1519 : VoidPtrTy;
1520 buildGlobalBlock(CGM, blockInfo,
1521 blockFn: llvm::ConstantExpr::getPointerCast(C: fn, Ty: GenVoidPtrTy));
1522 }
1523
1524 // Begin generating the function.
1525 StartFunction(GD: blockDecl, RetTy: fnType->getReturnType(), Fn: fn, FnInfo: fnInfo, Args: args,
1526 Loc: blockDecl->getLocation(),
1527 StartLoc: blockInfo.getBlockExpr()->getBody()->getBeginLoc());
1528
1529 // Okay. Undo some of what StartFunction did.
1530
1531 // At -O0 we generate an explicit alloca for the BlockPointer, so the RA
1532 // won't delete the dbg.declare intrinsics for captured variables.
1533 llvm::Value *BlockPointerDbgLoc = BlockPointer;
1534 if (CGM.getCodeGenOpts().OptimizationLevel == 0) {
1535 // Allocate a stack slot for it, so we can point the debugger to it
1536 Address Alloca = CreateTempAlloca(Ty: BlockPointer->getType(),
1537 align: getPointerAlign(),
1538 Name: "block.addr");
1539 // Set the DebugLocation to empty, so the store is recognized as a
1540 // frame setup instruction by llvm::DwarfDebug::beginFunction().
1541 auto NL = ApplyDebugLocation::CreateEmpty(CGF&: *this);
1542 Builder.CreateStore(Val: BlockPointer, Addr: Alloca);
1543 BlockPointerDbgLoc = Alloca.emitRawPointer(CGF&: *this);
1544 }
1545
1546 // If we have a C++ 'this' reference, go ahead and force it into
1547 // existence now.
1548 if (blockDecl->capturesCXXThis()) {
1549 Address addr = Builder.CreateStructGEP(
1550 Addr: LoadBlockStruct(), Index: blockInfo.CXXThisIndex, Name: "block.captured-this");
1551 CXXThisValue = Builder.CreateLoad(Addr: addr, Name: "this");
1552 }
1553
1554 // Also force all the constant captures.
1555 for (const auto &CI : blockDecl->captures()) {
1556 const VarDecl *variable = CI.getVariable();
1557 const CGBlockInfo::Capture &capture = blockInfo.getCapture(var: variable);
1558 if (!capture.isConstant()) continue;
1559
1560 CharUnits align = getContext().getDeclAlign(D: variable);
1561 Address alloca = CreateMemTempWithoutCast(T: variable->getType(), Align: align,
1562 Name: "block.captured-const");
1563
1564 Builder.CreateStore(Val: capture.getConstant(), Addr: alloca);
1565
1566 setAddrOfLocalVar(VD: variable, Addr: alloca);
1567 }
1568
1569 // Save a spot to insert the debug information for all the DeclRefExprs.
1570 llvm::BasicBlock *entry = Builder.GetInsertBlock();
1571 llvm::BasicBlock::iterator entry_ptr = Builder.GetInsertPoint();
1572 --entry_ptr;
1573
1574 if (IsLambdaConversionToBlock)
1575 EmitLambdaBlockInvokeBody();
1576 else {
1577 PGO->assignRegionCounters(GD: GlobalDecl(blockDecl), Fn: fn);
1578 incrementProfileCounter(S: blockDecl->getBody());
1579 maybeCreateMCDCCondBitmap();
1580 EmitStmt(S: blockDecl->getBody());
1581 }
1582
1583 // Remember where we were...
1584 llvm::BasicBlock *resume = Builder.GetInsertBlock();
1585
1586 // Go back to the entry.
1587 if (entry_ptr->getNextNode())
1588 entry_ptr = entry_ptr->getNextNode()->getIterator();
1589 else
1590 entry_ptr = entry->end();
1591 Builder.SetInsertPoint(TheBB: entry, IP: entry_ptr);
1592
1593 // Emit debug information for all the DeclRefExprs.
1594 // FIXME: also for 'this'
1595 if (CGDebugInfo *DI = getDebugInfo()) {
1596 for (const auto &CI : blockDecl->captures()) {
1597 const VarDecl *variable = CI.getVariable();
1598 DI->EmitLocation(Builder, Loc: variable->getLocation());
1599
1600 if (CGM.getCodeGenOpts().hasReducedDebugInfo()) {
1601 const CGBlockInfo::Capture &capture = blockInfo.getCapture(var: variable);
1602 if (capture.isConstant()) {
1603 auto addr = LocalDeclMap.find(Val: variable)->second;
1604 (void)DI->EmitDeclareOfAutoVariable(
1605 Decl: variable, AI: addr.emitRawPointer(CGF&: *this), Builder);
1606 continue;
1607 }
1608
1609 DI->EmitDeclareOfBlockDeclRefVariable(
1610 variable, storage: BlockPointerDbgLoc, Builder, blockInfo,
1611 InsertPoint: entry_ptr == entry->end() ? nullptr : &*entry_ptr);
1612 }
1613 }
1614 // Recover location if it was changed in the above loop.
1615 DI->EmitLocation(Builder,
1616 Loc: cast<CompoundStmt>(Val: blockDecl->getBody())->getRBracLoc());
1617 }
1618
1619 // And resume where we left off.
1620 if (resume == nullptr)
1621 Builder.ClearInsertionPoint();
1622 else
1623 Builder.SetInsertPoint(resume);
1624
1625 FinishFunction(EndLoc: cast<CompoundStmt>(Val: blockDecl->getBody())->getRBracLoc());
1626
1627 return fn;
1628}
1629
1630static std::pair<BlockCaptureEntityKind, BlockFieldFlags>
1631computeCopyInfoForBlockCapture(const BlockDecl::Capture &CI, QualType T,
1632 const LangOptions &LangOpts) {
1633 if (CI.getCopyExpr()) {
1634 assert(!CI.isByRef());
1635 // don't bother computing flags
1636 return std::make_pair(x: BlockCaptureEntityKind::CXXRecord, y: BlockFieldFlags());
1637 }
1638 BlockFieldFlags Flags;
1639 if (CI.isEscapingByref()) {
1640 Flags = BLOCK_FIELD_IS_BYREF;
1641 if (T.isObjCGCWeak())
1642 Flags |= BLOCK_FIELD_IS_WEAK;
1643 return std::make_pair(x: BlockCaptureEntityKind::BlockObject, y&: Flags);
1644 }
1645
1646 if (T.hasAddressDiscriminatedPointerAuth())
1647 return std::make_pair(
1648 x: BlockCaptureEntityKind::AddressDiscriminatedPointerAuth, y&: Flags);
1649
1650 Flags = BLOCK_FIELD_IS_OBJECT;
1651 bool isBlockPointer = T->isBlockPointerType();
1652 if (isBlockPointer)
1653 Flags = BLOCK_FIELD_IS_BLOCK;
1654
1655 switch (T.isNonTrivialToPrimitiveCopy()) {
1656 case QualType::PCK_Struct:
1657 return std::make_pair(x: BlockCaptureEntityKind::NonTrivialCStruct,
1658 y: BlockFieldFlags());
1659 case QualType::PCK_ARCWeak:
1660 // We need to register __weak direct captures with the runtime.
1661 return std::make_pair(x: BlockCaptureEntityKind::ARCWeak, y&: Flags);
1662 case QualType::PCK_ARCStrong:
1663 // We need to retain the copied value for __strong direct captures.
1664 // If it's a block pointer, we have to copy the block and assign that to
1665 // the destination pointer, so we might as well use _Block_object_assign.
1666 // Otherwise we can avoid that.
1667 return std::make_pair(x: !isBlockPointer ? BlockCaptureEntityKind::ARCStrong
1668 : BlockCaptureEntityKind::BlockObject,
1669 y&: Flags);
1670 case QualType::PCK_PtrAuth:
1671 return std::make_pair(
1672 x: BlockCaptureEntityKind::AddressDiscriminatedPointerAuth,
1673 y: BlockFieldFlags());
1674 case QualType::PCK_Trivial:
1675 case QualType::PCK_VolatileTrivial: {
1676 if (!T->isObjCRetainableType())
1677 // For all other types, the memcpy is fine.
1678 return std::make_pair(x: BlockCaptureEntityKind::None, y: BlockFieldFlags());
1679
1680 // Honor the inert __unsafe_unretained qualifier, which doesn't actually
1681 // make it into the type system.
1682 if (T->isObjCInertUnsafeUnretainedType())
1683 return std::make_pair(x: BlockCaptureEntityKind::None, y: BlockFieldFlags());
1684
1685 // Special rules for ARC captures:
1686 Qualifiers QS = T.getQualifiers();
1687
1688 // Non-ARC captures of retainable pointers are strong and
1689 // therefore require a call to _Block_object_assign.
1690 if (!QS.getObjCLifetime() && !LangOpts.ObjCAutoRefCount)
1691 return std::make_pair(x: BlockCaptureEntityKind::BlockObject, y&: Flags);
1692
1693 // Otherwise the memcpy is fine.
1694 return std::make_pair(x: BlockCaptureEntityKind::None, y: BlockFieldFlags());
1695 }
1696 }
1697 llvm_unreachable("after exhaustive PrimitiveCopyKind switch");
1698}
1699
1700namespace {
1701/// Release a __block variable.
1702struct CallBlockRelease final : EHScopeStack::Cleanup {
1703 Address Addr;
1704 BlockFieldFlags FieldFlags;
1705 bool LoadBlockVarAddr, CanThrow;
1706
1707 CallBlockRelease(Address Addr, BlockFieldFlags Flags, bool LoadValue,
1708 bool CT)
1709 : Addr(Addr), FieldFlags(Flags), LoadBlockVarAddr(LoadValue),
1710 CanThrow(CT) {}
1711
1712 void Emit(CodeGenFunction &CGF, Flags flags) override {
1713 llvm::Value *BlockVarAddr;
1714 if (LoadBlockVarAddr) {
1715 BlockVarAddr = CGF.Builder.CreateLoad(Addr);
1716 } else {
1717 BlockVarAddr = Addr.emitRawPointer(CGF);
1718 }
1719
1720 CGF.BuildBlockRelease(DeclPtr: BlockVarAddr, flags: FieldFlags, CanThrow);
1721 }
1722};
1723} // end anonymous namespace
1724
1725/// Check if \p T is a C++ class that has a destructor that can throw.
1726bool CodeGenFunction::cxxDestructorCanThrow(QualType T) {
1727 if (const auto *RD = T->getAsCXXRecordDecl())
1728 if (const CXXDestructorDecl *DD = RD->getDestructor())
1729 return DD->getType()->castAs<FunctionProtoType>()->canThrow();
1730 return false;
1731}
1732
1733// Return a string that has the information about a capture.
1734static std::string getBlockCaptureStr(const CGBlockInfo::Capture &Cap,
1735 CaptureStrKind StrKind,
1736 CharUnits BlockAlignment,
1737 CodeGenModule &CGM) {
1738 std::string Str;
1739 ASTContext &Ctx = CGM.getContext();
1740 const BlockDecl::Capture &CI = *Cap.Cap;
1741 QualType CaptureTy = CI.getVariable()->getType();
1742
1743 BlockCaptureEntityKind Kind;
1744 BlockFieldFlags Flags;
1745
1746 // CaptureStrKind::Merged should be passed only when the operations and the
1747 // flags are the same for copy and dispose.
1748 assert((StrKind != CaptureStrKind::Merged ||
1749 (Cap.CopyKind == Cap.DisposeKind &&
1750 Cap.CopyFlags == Cap.DisposeFlags)) &&
1751 "different operations and flags");
1752
1753 if (StrKind == CaptureStrKind::DisposeHelper) {
1754 Kind = Cap.DisposeKind;
1755 Flags = Cap.DisposeFlags;
1756 } else {
1757 Kind = Cap.CopyKind;
1758 Flags = Cap.CopyFlags;
1759 }
1760
1761 switch (Kind) {
1762 case BlockCaptureEntityKind::CXXRecord: {
1763 Str += "c";
1764 SmallString<256> TyStr;
1765 llvm::raw_svector_ostream Out(TyStr);
1766 CGM.getCXXABI().getMangleContext().mangleCanonicalTypeName(T: CaptureTy, Out);
1767 Str += llvm::to_string(Value: TyStr.size()) + TyStr.c_str();
1768 break;
1769 }
1770 case BlockCaptureEntityKind::ARCWeak:
1771 Str += "w";
1772 break;
1773 case BlockCaptureEntityKind::ARCStrong:
1774 Str += "s";
1775 break;
1776 case BlockCaptureEntityKind::AddressDiscriminatedPointerAuth: {
1777 auto PtrAuth = CaptureTy.getPointerAuth();
1778 assert(PtrAuth && PtrAuth.isAddressDiscriminated());
1779 Str += "p" + llvm::to_string(Value: PtrAuth.getKey()) + "d" +
1780 llvm::to_string(Value: PtrAuth.getExtraDiscriminator());
1781 break;
1782 }
1783 case BlockCaptureEntityKind::BlockObject: {
1784 const VarDecl *Var = CI.getVariable();
1785 unsigned F = Flags.getBitMask();
1786 if (F & BLOCK_FIELD_IS_BYREF) {
1787 Str += "r";
1788 if (F & BLOCK_FIELD_IS_WEAK)
1789 Str += "w";
1790 else {
1791 // If CaptureStrKind::Merged is passed, check both the copy expression
1792 // and the destructor.
1793 if (StrKind != CaptureStrKind::DisposeHelper) {
1794 if (Ctx.getBlockVarCopyInit(VD: Var).canThrow())
1795 Str += "c";
1796 }
1797 if (StrKind != CaptureStrKind::CopyHelper) {
1798 if (CodeGenFunction::cxxDestructorCanThrow(T: CaptureTy))
1799 Str += "d";
1800 }
1801 }
1802 } else {
1803 assert((F & BLOCK_FIELD_IS_OBJECT) && "unexpected flag value");
1804 if (F == BLOCK_FIELD_IS_BLOCK)
1805 Str += "b";
1806 else
1807 Str += "o";
1808 }
1809 break;
1810 }
1811 case BlockCaptureEntityKind::NonTrivialCStruct: {
1812 bool IsVolatile = CaptureTy.isVolatileQualified();
1813 CharUnits Alignment = BlockAlignment.alignmentAtOffset(offset: Cap.getOffset());
1814
1815 Str += "n";
1816 std::string FuncStr;
1817 if (StrKind == CaptureStrKind::DisposeHelper)
1818 FuncStr = CodeGenFunction::getNonTrivialDestructorStr(
1819 QT: CaptureTy, Alignment, IsVolatile, Ctx);
1820 else
1821 // If CaptureStrKind::Merged is passed, use the copy constructor string.
1822 // It has all the information that the destructor string has.
1823 FuncStr = CodeGenFunction::getNonTrivialCopyConstructorStr(
1824 QT: CaptureTy, Alignment, IsVolatile, Ctx);
1825 // The underscore is necessary here because non-trivial copy constructor
1826 // and destructor strings can start with a number.
1827 Str += llvm::to_string(Value: FuncStr.size()) + "_" + FuncStr;
1828 break;
1829 }
1830 case BlockCaptureEntityKind::None:
1831 break;
1832 }
1833
1834 return Str;
1835}
1836
1837static std::string getCopyDestroyHelperFuncName(
1838 const SmallVectorImpl<CGBlockInfo::Capture> &Captures,
1839 CharUnits BlockAlignment, CaptureStrKind StrKind, CodeGenModule &CGM) {
1840 assert((StrKind == CaptureStrKind::CopyHelper ||
1841 StrKind == CaptureStrKind::DisposeHelper) &&
1842 "unexpected CaptureStrKind");
1843 std::string Name = StrKind == CaptureStrKind::CopyHelper
1844 ? "__copy_helper_block_"
1845 : "__destroy_helper_block_";
1846 if (CGM.getLangOpts().Exceptions)
1847 Name += "e";
1848 if (CGM.getCodeGenOpts().ObjCAutoRefCountExceptions)
1849 Name += "a";
1850 Name += llvm::to_string(Value: BlockAlignment.getQuantity()) + "_";
1851
1852 for (auto &Cap : Captures) {
1853 if (Cap.isConstantOrTrivial())
1854 continue;
1855 Name += llvm::to_string(Value: Cap.getOffset().getQuantity());
1856 Name += getBlockCaptureStr(Cap, StrKind, BlockAlignment, CGM);
1857 }
1858
1859 return Name;
1860}
1861
1862static void pushCaptureCleanup(BlockCaptureEntityKind CaptureKind,
1863 Address Field, QualType CaptureType,
1864 BlockFieldFlags Flags, bool ForCopyHelper,
1865 VarDecl *Var, CodeGenFunction &CGF) {
1866 bool EHOnly = ForCopyHelper;
1867
1868 switch (CaptureKind) {
1869 case BlockCaptureEntityKind::CXXRecord:
1870 case BlockCaptureEntityKind::ARCWeak:
1871 case BlockCaptureEntityKind::NonTrivialCStruct:
1872 case BlockCaptureEntityKind::ARCStrong: {
1873 if (CaptureType.isDestructedType() &&
1874 (!EHOnly || CGF.needsEHCleanup(kind: CaptureType.isDestructedType()))) {
1875 CodeGenFunction::Destroyer *Destroyer =
1876 CaptureKind == BlockCaptureEntityKind::ARCStrong
1877 ? CodeGenFunction::destroyARCStrongImprecise
1878 : CGF.getDestroyer(destructionKind: CaptureType.isDestructedType());
1879 CleanupKind Kind =
1880 EHOnly ? EHCleanup
1881 : CGF.getCleanupKind(kind: CaptureType.isDestructedType());
1882 CGF.pushDestroy(kind: Kind, addr: Field, type: CaptureType, destroyer: Destroyer, useEHCleanupForArray: Kind & EHCleanup);
1883 }
1884 break;
1885 }
1886 case BlockCaptureEntityKind::BlockObject: {
1887 if (!EHOnly || CGF.getLangOpts().Exceptions) {
1888 CleanupKind Kind = EHOnly ? EHCleanup : NormalAndEHCleanup;
1889 // Calls to _Block_object_dispose along the EH path in the copy helper
1890 // function don't throw as newly-copied __block variables always have a
1891 // reference count of 2.
1892 bool CanThrow =
1893 !ForCopyHelper && CGF.cxxDestructorCanThrow(T: CaptureType);
1894 CGF.enterByrefCleanup(Kind, Addr: Field, Flags, /*LoadBlockVarAddr*/ true,
1895 CanThrow);
1896 }
1897 break;
1898 }
1899 case BlockCaptureEntityKind::AddressDiscriminatedPointerAuth:
1900 case BlockCaptureEntityKind::None:
1901 break;
1902 }
1903}
1904
1905static void setBlockHelperAttributesVisibility(bool CapturesNonExternalType,
1906 llvm::Function *Fn,
1907 const CGFunctionInfo &FI,
1908 CodeGenModule &CGM) {
1909 if (CapturesNonExternalType) {
1910 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI);
1911 } else {
1912 Fn->setVisibility(llvm::GlobalValue::HiddenVisibility);
1913 Fn->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
1914 CGM.SetLLVMFunctionAttributes(GD: GlobalDecl(), Info: FI, F: Fn, /*IsThunk=*/false);
1915 CGM.SetLLVMFunctionAttributesForDefinition(D: nullptr, F: Fn);
1916 }
1917}
1918/// Generate the copy-helper function for a block closure object:
1919/// static void block_copy_helper(block_t *dst, block_t *src);
1920/// The runtime will have previously initialized 'dst' by doing a
1921/// bit-copy of 'src'.
1922///
1923/// Note that this copies an entire block closure object to the heap;
1924/// it should not be confused with a 'byref copy helper', which moves
1925/// the contents of an individual __block variable to the heap.
1926llvm::Constant *
1927CodeGenFunction::GenerateCopyHelperFunction(const CGBlockInfo &blockInfo) {
1928 std::string FuncName = getCopyDestroyHelperFuncName(
1929 Captures: blockInfo.SortedCaptures, BlockAlignment: blockInfo.BlockAlign,
1930 StrKind: CaptureStrKind::CopyHelper, CGM);
1931
1932 if (llvm::GlobalValue *Func = CGM.getModule().getNamedValue(Name: FuncName))
1933 return Func;
1934
1935 ASTContext &C = getContext();
1936
1937 QualType ReturnTy = C.VoidTy;
1938
1939 auto *DstDecl =
1940 ImplicitParamDecl::Create(C, T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
1941 auto *SrcDecl =
1942 ImplicitParamDecl::Create(C, T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
1943
1944 FunctionArgList args{DstDecl, SrcDecl};
1945 const CGFunctionInfo &FI =
1946 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: ReturnTy, args);
1947
1948 // FIXME: it would be nice if these were mergeable with things with
1949 // identical semantics.
1950 llvm::FunctionType *LTy = CGM.getTypes().GetFunctionType(Info: FI);
1951
1952 llvm::Function *Fn =
1953 llvm::Function::Create(Ty: LTy, Linkage: llvm::GlobalValue::LinkOnceODRLinkage,
1954 N: FuncName, M: &CGM.getModule());
1955 if (CGM.supportsCOMDAT())
1956 Fn->setComdat(CGM.getModule().getOrInsertComdat(Name: FuncName));
1957
1958 setBlockHelperAttributesVisibility(CapturesNonExternalType: blockInfo.CapturesNonExternalType, Fn, FI,
1959 CGM);
1960 StartFunction(GD: GlobalDecl(), RetTy: ReturnTy, Fn, FnInfo: FI, Args: args);
1961 auto AL = ApplyDebugLocation::CreateArtificial(CGF&: *this);
1962
1963 Address src = GetAddrOfLocalVar(VD: SrcDecl);
1964 src = Address(Builder.CreateLoad(Addr: src), blockInfo.StructureType,
1965 blockInfo.BlockAlign);
1966
1967 Address dst = GetAddrOfLocalVar(VD: DstDecl);
1968 dst = Address(Builder.CreateLoad(Addr: dst), blockInfo.StructureType,
1969 blockInfo.BlockAlign);
1970
1971 for (auto &capture : blockInfo.SortedCaptures) {
1972 if (capture.isConstantOrTrivial())
1973 continue;
1974
1975 const BlockDecl::Capture &CI = *capture.Cap;
1976 QualType captureType = CI.getVariable()->getType();
1977 BlockFieldFlags flags = capture.CopyFlags;
1978
1979 unsigned index = capture.getIndex();
1980 Address srcField = Builder.CreateStructGEP(Addr: src, Index: index);
1981 Address dstField = Builder.CreateStructGEP(Addr: dst, Index: index);
1982
1983 switch (capture.CopyKind) {
1984 case BlockCaptureEntityKind::CXXRecord:
1985 // If there's an explicit copy expression, we do that.
1986 assert(CI.getCopyExpr() && "copy expression for variable is missing");
1987 EmitSynthesizedCXXCopyCtor(Dest: dstField, Src: srcField, Exp: CI.getCopyExpr());
1988 break;
1989 case BlockCaptureEntityKind::ARCWeak:
1990 EmitARCCopyWeak(dst: dstField, src: srcField);
1991 break;
1992 case BlockCaptureEntityKind::AddressDiscriminatedPointerAuth: {
1993 QualType Type = CI.getVariable()->getType();
1994 PointerAuthQualifier PointerAuth = Type.getPointerAuth();
1995 assert(PointerAuth && PointerAuth.isAddressDiscriminated());
1996 EmitPointerAuthCopy(Qualifier: PointerAuth, Type, DestField: dstField, SrcField: srcField);
1997 // We don't need to push cleanups for ptrauth types.
1998 continue;
1999 }
2000 case BlockCaptureEntityKind::NonTrivialCStruct: {
2001 // If this is a C struct that requires non-trivial copy construction,
2002 // emit a call to its copy constructor.
2003 QualType varType = CI.getVariable()->getType();
2004 callCStructCopyConstructor(Dst: MakeAddrLValue(Addr: dstField, T: varType),
2005 Src: MakeAddrLValue(Addr: srcField, T: varType));
2006 break;
2007 }
2008 case BlockCaptureEntityKind::ARCStrong: {
2009 llvm::Value *srcValue = Builder.CreateLoad(Addr: srcField, Name: "blockcopy.src");
2010 // At -O0, store null into the destination field (so that the
2011 // storeStrong doesn't over-release) and then call storeStrong.
2012 // This is a workaround to not having an initStrong call.
2013 if (CGM.getCodeGenOpts().OptimizationLevel == 0) {
2014 auto *ty = cast<llvm::PointerType>(Val: srcValue->getType());
2015 llvm::Value *null = llvm::ConstantPointerNull::get(T: ty);
2016 Builder.CreateStore(Val: null, Addr: dstField);
2017 EmitARCStoreStrongCall(addr: dstField, value: srcValue, resultIgnored: true);
2018
2019 // With optimization enabled, take advantage of the fact that
2020 // the blocks runtime guarantees a memcpy of the block data, and
2021 // just emit a retain of the src field.
2022 } else {
2023 EmitARCRetainNonBlock(value: srcValue);
2024
2025 // Unless EH cleanup is required, we don't need this anymore, so kill
2026 // it. It's not quite worth the annoyance to avoid creating it in the
2027 // first place.
2028 if (!needsEHCleanup(kind: captureType.isDestructedType()))
2029 if (auto *I = cast_or_null<llvm::Instruction>(
2030 Val: dstField.getPointerIfNotSigned()))
2031 I->eraseFromParent();
2032 }
2033 break;
2034 }
2035 case BlockCaptureEntityKind::BlockObject: {
2036 llvm::Value *srcValue = Builder.CreateLoad(Addr: srcField, Name: "blockcopy.src");
2037 llvm::Value *dstAddr = dstField.emitRawPointer(CGF&: *this);
2038 llvm::Value *args[] = {
2039 dstAddr, srcValue, llvm::ConstantInt::get(Ty: Int32Ty, V: flags.getBitMask())
2040 };
2041
2042 if (CI.isByRef() && C.getBlockVarCopyInit(VD: CI.getVariable()).canThrow())
2043 EmitRuntimeCallOrInvoke(callee: CGM.getBlockObjectAssign(), args);
2044 else
2045 EmitNounwindRuntimeCall(callee: CGM.getBlockObjectAssign(), args);
2046 break;
2047 }
2048 case BlockCaptureEntityKind::None:
2049 continue;
2050 }
2051
2052 // Ensure that we destroy the copied object if an exception is thrown later
2053 // in the helper function.
2054 pushCaptureCleanup(CaptureKind: capture.CopyKind, Field: dstField, CaptureType: captureType, Flags: flags,
2055 /*ForCopyHelper*/ true, Var: CI.getVariable(), CGF&: *this);
2056 }
2057
2058 FinishFunction();
2059
2060 return Fn;
2061}
2062
2063static BlockFieldFlags
2064getBlockFieldFlagsForObjCObjectPointer(const BlockDecl::Capture &CI,
2065 QualType T) {
2066 BlockFieldFlags Flags = BLOCK_FIELD_IS_OBJECT;
2067 if (T->isBlockPointerType())
2068 Flags = BLOCK_FIELD_IS_BLOCK;
2069 return Flags;
2070}
2071
2072static std::pair<BlockCaptureEntityKind, BlockFieldFlags>
2073computeDestroyInfoForBlockCapture(const BlockDecl::Capture &CI, QualType T,
2074 const LangOptions &LangOpts) {
2075 if (CI.isEscapingByref()) {
2076 BlockFieldFlags Flags = BLOCK_FIELD_IS_BYREF;
2077 if (T.isObjCGCWeak())
2078 Flags |= BLOCK_FIELD_IS_WEAK;
2079 return std::make_pair(x: BlockCaptureEntityKind::BlockObject, y&: Flags);
2080 }
2081
2082 switch (T.isDestructedType()) {
2083 case QualType::DK_cxx_destructor:
2084 return std::make_pair(x: BlockCaptureEntityKind::CXXRecord, y: BlockFieldFlags());
2085 case QualType::DK_objc_strong_lifetime:
2086 // Use objc_storeStrong for __strong direct captures; the
2087 // dynamic tools really like it when we do this.
2088 return std::make_pair(x: BlockCaptureEntityKind::ARCStrong,
2089 y: getBlockFieldFlagsForObjCObjectPointer(CI, T));
2090 case QualType::DK_objc_weak_lifetime:
2091 // Support __weak direct captures.
2092 return std::make_pair(x: BlockCaptureEntityKind::ARCWeak,
2093 y: getBlockFieldFlagsForObjCObjectPointer(CI, T));
2094 case QualType::DK_nontrivial_c_struct:
2095 return std::make_pair(x: BlockCaptureEntityKind::NonTrivialCStruct,
2096 y: BlockFieldFlags());
2097 case QualType::DK_none: {
2098 // Non-ARC captures are strong, and we need to use _Block_object_dispose.
2099 // But honor the inert __unsafe_unretained qualifier, which doesn't actually
2100 // make it into the type system.
2101 if (T->isObjCRetainableType() && !T.getQualifiers().hasObjCLifetime() &&
2102 !LangOpts.ObjCAutoRefCount && !T->isObjCInertUnsafeUnretainedType())
2103 return std::make_pair(x: BlockCaptureEntityKind::BlockObject,
2104 y: getBlockFieldFlagsForObjCObjectPointer(CI, T));
2105 // Otherwise, we have nothing to do.
2106 return std::make_pair(x: BlockCaptureEntityKind::None, y: BlockFieldFlags());
2107 }
2108 }
2109 llvm_unreachable("after exhaustive DestructionKind switch");
2110}
2111
2112/// Generate the destroy-helper function for a block closure object:
2113/// static void block_destroy_helper(block_t *theBlock);
2114///
2115/// Note that this destroys a heap-allocated block closure object;
2116/// it should not be confused with a 'byref destroy helper', which
2117/// destroys the heap-allocated contents of an individual __block
2118/// variable.
2119llvm::Constant *
2120CodeGenFunction::GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo) {
2121 std::string FuncName = getCopyDestroyHelperFuncName(
2122 Captures: blockInfo.SortedCaptures, BlockAlignment: blockInfo.BlockAlign,
2123 StrKind: CaptureStrKind::DisposeHelper, CGM);
2124
2125 if (llvm::GlobalValue *Func = CGM.getModule().getNamedValue(Name: FuncName))
2126 return Func;
2127
2128 ASTContext &C = getContext();
2129
2130 QualType ReturnTy = C.VoidTy;
2131
2132 auto *SrcDecl =
2133 ImplicitParamDecl::Create(C, T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
2134
2135 FunctionArgList args{SrcDecl};
2136 const CGFunctionInfo &FI =
2137 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: ReturnTy, args);
2138
2139 // FIXME: We'd like to put these into a mergable by content, with
2140 // internal linkage.
2141 llvm::FunctionType *LTy = CGM.getTypes().GetFunctionType(Info: FI);
2142
2143 llvm::Function *Fn =
2144 llvm::Function::Create(Ty: LTy, Linkage: llvm::GlobalValue::LinkOnceODRLinkage,
2145 N: FuncName, M: &CGM.getModule());
2146 if (CGM.supportsCOMDAT())
2147 Fn->setComdat(CGM.getModule().getOrInsertComdat(Name: FuncName));
2148
2149 setBlockHelperAttributesVisibility(CapturesNonExternalType: blockInfo.CapturesNonExternalType, Fn, FI,
2150 CGM);
2151 StartFunction(GD: GlobalDecl(), RetTy: ReturnTy, Fn, FnInfo: FI, Args: args);
2152 markAsIgnoreThreadCheckingAtRuntime(Fn);
2153
2154 auto AL = ApplyDebugLocation::CreateArtificial(CGF&: *this);
2155
2156 Address src = GetAddrOfLocalVar(VD: SrcDecl);
2157 src = Address(Builder.CreateLoad(Addr: src), blockInfo.StructureType,
2158 blockInfo.BlockAlign);
2159
2160 CodeGenFunction::RunCleanupsScope cleanups(*this);
2161
2162 for (auto &capture : blockInfo.SortedCaptures) {
2163 if (capture.isConstantOrTrivial())
2164 continue;
2165
2166 const BlockDecl::Capture &CI = *capture.Cap;
2167 BlockFieldFlags flags = capture.DisposeFlags;
2168
2169 Address srcField = Builder.CreateStructGEP(Addr: src, Index: capture.getIndex());
2170
2171 pushCaptureCleanup(CaptureKind: capture.DisposeKind, Field: srcField,
2172 CaptureType: CI.getVariable()->getType(), Flags: flags,
2173 /*ForCopyHelper*/ false, Var: CI.getVariable(), CGF&: *this);
2174 }
2175
2176 cleanups.ForceCleanup();
2177
2178 FinishFunction();
2179
2180 return Fn;
2181}
2182
2183namespace {
2184
2185/// Emits the copy/dispose helper functions for a __block object of id type.
2186class ObjectByrefHelpers final : public BlockByrefHelpers {
2187 BlockFieldFlags Flags;
2188
2189public:
2190 ObjectByrefHelpers(CharUnits alignment, BlockFieldFlags flags)
2191 : BlockByrefHelpers(alignment), Flags(flags) {}
2192
2193 void emitCopy(CodeGenFunction &CGF, Address destField,
2194 Address srcField) override {
2195 destField = destField.withElementType(ElemTy: CGF.Int8Ty);
2196
2197 srcField = srcField.withElementType(ElemTy: CGF.Int8PtrTy);
2198 llvm::Value *srcValue = CGF.Builder.CreateLoad(Addr: srcField);
2199
2200 unsigned flags = (Flags | BLOCK_BYREF_CALLER).getBitMask();
2201
2202 llvm::Value *flagsVal = llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: flags);
2203 llvm::FunctionCallee fn = CGF.CGM.getBlockObjectAssign();
2204
2205 llvm::Value *args[] = {destField.emitRawPointer(CGF), srcValue, flagsVal};
2206 CGF.EmitNounwindRuntimeCall(callee: fn, args);
2207 }
2208
2209 void emitDispose(CodeGenFunction &CGF, Address field) override {
2210 field = field.withElementType(ElemTy: CGF.Int8PtrTy);
2211 llvm::Value *value = CGF.Builder.CreateLoad(Addr: field);
2212
2213 CGF.BuildBlockRelease(DeclPtr: value, flags: Flags | BLOCK_BYREF_CALLER, CanThrow: false);
2214 }
2215
2216 void profileImpl(llvm::FoldingSetNodeID &id) const override {
2217 id.AddInteger(I: Flags.getBitMask());
2218 }
2219};
2220
2221/// Emits the copy/dispose helpers for an ARC __block __weak variable.
2222class ARCWeakByrefHelpers final : public BlockByrefHelpers {
2223public:
2224 ARCWeakByrefHelpers(CharUnits alignment) : BlockByrefHelpers(alignment) {}
2225
2226 void emitCopy(CodeGenFunction &CGF, Address destField,
2227 Address srcField) override {
2228 CGF.EmitARCMoveWeak(dst: destField, src: srcField);
2229 }
2230
2231 void emitDispose(CodeGenFunction &CGF, Address field) override {
2232 CGF.EmitARCDestroyWeak(addr: field);
2233 }
2234
2235 void profileImpl(llvm::FoldingSetNodeID &id) const override {
2236 // 0 is distinguishable from all pointers and byref flags
2237 id.AddInteger(I: 0);
2238 }
2239};
2240
2241/// Emits the copy/dispose helpers for an ARC __block __strong variable
2242/// that's not of block-pointer type.
2243class ARCStrongByrefHelpers final : public BlockByrefHelpers {
2244public:
2245 ARCStrongByrefHelpers(CharUnits alignment) : BlockByrefHelpers(alignment) {}
2246
2247 void emitCopy(CodeGenFunction &CGF, Address destField,
2248 Address srcField) override {
2249 // Do a "move" by copying the value and then zeroing out the old
2250 // variable.
2251
2252 llvm::Value *value = CGF.Builder.CreateLoad(Addr: srcField);
2253
2254 llvm::Value *null =
2255 llvm::ConstantPointerNull::get(T: cast<llvm::PointerType>(Val: value->getType()));
2256
2257 if (CGF.CGM.getCodeGenOpts().OptimizationLevel == 0) {
2258 CGF.Builder.CreateStore(Val: null, Addr: destField);
2259 CGF.EmitARCStoreStrongCall(addr: destField, value, /*ignored*/ resultIgnored: true);
2260 CGF.EmitARCStoreStrongCall(addr: srcField, value: null, /*ignored*/ resultIgnored: true);
2261 return;
2262 }
2263 CGF.Builder.CreateStore(Val: value, Addr: destField);
2264 CGF.Builder.CreateStore(Val: null, Addr: srcField);
2265 }
2266
2267 void emitDispose(CodeGenFunction &CGF, Address field) override {
2268 CGF.EmitARCDestroyStrong(addr: field, precise: ARCImpreciseLifetime);
2269 }
2270
2271 void profileImpl(llvm::FoldingSetNodeID &id) const override {
2272 // 1 is distinguishable from all pointers and byref flags
2273 id.AddInteger(I: 1);
2274 }
2275};
2276
2277/// Emits the copy/dispose helpers for an ARC __block __strong
2278/// variable that's of block-pointer type.
2279class ARCStrongBlockByrefHelpers final : public BlockByrefHelpers {
2280public:
2281 ARCStrongBlockByrefHelpers(CharUnits alignment)
2282 : BlockByrefHelpers(alignment) {}
2283
2284 void emitCopy(CodeGenFunction &CGF, Address destField,
2285 Address srcField) override {
2286 // Do the copy with objc_retainBlock; that's all that
2287 // _Block_object_assign would do anyway, and we'd have to pass the
2288 // right arguments to make sure it doesn't get no-op'ed.
2289 llvm::Value *oldValue = CGF.Builder.CreateLoad(Addr: srcField);
2290 llvm::Value *copy = CGF.EmitARCRetainBlock(value: oldValue, /*mandatory*/ true);
2291 CGF.Builder.CreateStore(Val: copy, Addr: destField);
2292 }
2293
2294 void emitDispose(CodeGenFunction &CGF, Address field) override {
2295 CGF.EmitARCDestroyStrong(addr: field, precise: ARCImpreciseLifetime);
2296 }
2297
2298 void profileImpl(llvm::FoldingSetNodeID &id) const override {
2299 // 2 is distinguishable from all pointers and byref flags
2300 id.AddInteger(I: 2);
2301 }
2302};
2303
2304/// Emits the copy/dispose helpers for a __block variable with a
2305/// nontrivial copy constructor or destructor.
2306class CXXByrefHelpers final : public BlockByrefHelpers {
2307 QualType VarType;
2308 const Expr *CopyExpr;
2309
2310public:
2311 CXXByrefHelpers(CharUnits alignment, QualType type,
2312 const Expr *copyExpr)
2313 : BlockByrefHelpers(alignment), VarType(type), CopyExpr(copyExpr) {}
2314
2315 bool needsCopy() const override { return CopyExpr != nullptr; }
2316 void emitCopy(CodeGenFunction &CGF, Address destField,
2317 Address srcField) override {
2318 if (!CopyExpr) return;
2319 CGF.EmitSynthesizedCXXCopyCtor(Dest: destField, Src: srcField, Exp: CopyExpr);
2320 }
2321
2322 void emitDispose(CodeGenFunction &CGF, Address field) override {
2323 EHScopeStack::stable_iterator cleanupDepth = CGF.EHStack.stable_begin();
2324 CGF.PushDestructorCleanup(T: VarType, Addr: field);
2325 CGF.PopCleanupBlocks(OldCleanupStackSize: cleanupDepth);
2326 }
2327
2328 void profileImpl(llvm::FoldingSetNodeID &id) const override {
2329 id.AddPointer(Ptr: VarType.getCanonicalType().getAsOpaquePtr());
2330 }
2331};
2332
2333/// Emits the copy/dispose helpers for a __block variable with
2334/// address-discriminated pointer authentication.
2335class AddressDiscriminatedByrefHelpers final : public BlockByrefHelpers {
2336 QualType VarType;
2337
2338public:
2339 AddressDiscriminatedByrefHelpers(CharUnits Alignment, QualType Type)
2340 : BlockByrefHelpers(Alignment), VarType(Type) {
2341 assert(Type.hasAddressDiscriminatedPointerAuth());
2342 }
2343
2344 void emitCopy(CodeGenFunction &CGF, Address DestField,
2345 Address SrcField) override {
2346 CGF.EmitPointerAuthCopy(Qualifier: VarType.getPointerAuth(), Type: VarType, DestField,
2347 SrcField);
2348 }
2349
2350 bool needsDispose() const override { return false; }
2351 void emitDispose(CodeGenFunction &CGF, Address Field) override {
2352 llvm_unreachable("should never be called");
2353 }
2354
2355 void profileImpl(llvm::FoldingSetNodeID &ID) const override {
2356 ID.AddPointer(Ptr: VarType.getCanonicalType().getAsOpaquePtr());
2357 }
2358};
2359
2360/// Emits the copy/dispose helpers for a __block variable that is a non-trivial
2361/// C struct.
2362class NonTrivialCStructByrefHelpers final : public BlockByrefHelpers {
2363 QualType VarType;
2364
2365public:
2366 NonTrivialCStructByrefHelpers(CharUnits alignment, QualType type)
2367 : BlockByrefHelpers(alignment), VarType(type) {}
2368
2369 void emitCopy(CodeGenFunction &CGF, Address destField,
2370 Address srcField) override {
2371 CGF.callCStructMoveConstructor(Dst: CGF.MakeAddrLValue(Addr: destField, T: VarType),
2372 Src: CGF.MakeAddrLValue(Addr: srcField, T: VarType));
2373 }
2374
2375 bool needsDispose() const override {
2376 return VarType.isDestructedType();
2377 }
2378
2379 void emitDispose(CodeGenFunction &CGF, Address field) override {
2380 EHScopeStack::stable_iterator cleanupDepth = CGF.EHStack.stable_begin();
2381 CGF.pushDestroy(dtorKind: VarType.isDestructedType(), addr: field, type: VarType);
2382 CGF.PopCleanupBlocks(OldCleanupStackSize: cleanupDepth);
2383 }
2384
2385 void profileImpl(llvm::FoldingSetNodeID &id) const override {
2386 id.AddPointer(Ptr: VarType.getCanonicalType().getAsOpaquePtr());
2387 }
2388};
2389} // end anonymous namespace
2390
2391static llvm::Constant *
2392generateByrefCopyHelper(CodeGenFunction &CGF, const BlockByrefInfo &byrefInfo,
2393 BlockByrefHelpers &generator) {
2394 ASTContext &Context = CGF.getContext();
2395
2396 QualType ReturnTy = Context.VoidTy;
2397
2398 auto *Dst = ImplicitParamDecl::Create(C&: Context, T: Context.VoidPtrTy,
2399 ParamKind: ImplicitParamKind::Other);
2400 auto *Src = ImplicitParamDecl::Create(C&: Context, T: Context.VoidPtrTy,
2401 ParamKind: ImplicitParamKind::Other);
2402
2403 FunctionArgList args{Dst, Src};
2404 const CGFunctionInfo &FI =
2405 CGF.CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: ReturnTy, args);
2406
2407 llvm::FunctionType *LTy = CGF.CGM.getTypes().GetFunctionType(Info: FI);
2408
2409 // FIXME: We'd like to put these into a mergable by content, with
2410 // internal linkage.
2411 llvm::Function *Fn =
2412 llvm::Function::Create(Ty: LTy, Linkage: llvm::GlobalValue::InternalLinkage,
2413 N: "__Block_byref_object_copy_", M: &CGF.CGM.getModule());
2414
2415 CGF.CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI);
2416
2417 CGF.StartFunction(GD: GlobalDecl(), RetTy: ReturnTy, Fn, FnInfo: FI, Args: args);
2418 // Create a scope with an artificial location for the body of this function.
2419 auto AL = ApplyDebugLocation::CreateArtificial(CGF);
2420
2421 if (generator.needsCopy()) {
2422 // dst->x
2423 Address destField = CGF.GetAddrOfLocalVar(VD: Dst);
2424 destField = Address(CGF.Builder.CreateLoad(Addr: destField), byrefInfo.Type,
2425 byrefInfo.ByrefAlignment);
2426 destField =
2427 CGF.emitBlockByrefAddress(baseAddr: destField, info: byrefInfo, followForward: false, name: "dest-object");
2428
2429 // src->x
2430 Address srcField = CGF.GetAddrOfLocalVar(VD: Src);
2431 srcField = Address(CGF.Builder.CreateLoad(Addr: srcField), byrefInfo.Type,
2432 byrefInfo.ByrefAlignment);
2433 srcField =
2434 CGF.emitBlockByrefAddress(baseAddr: srcField, info: byrefInfo, followForward: false, name: "src-object");
2435
2436 generator.emitCopy(CGF, dest: destField, src: srcField);
2437 }
2438
2439 CGF.FinishFunction();
2440
2441 return Fn;
2442}
2443
2444/// Build the copy helper for a __block variable.
2445static llvm::Constant *buildByrefCopyHelper(CodeGenModule &CGM,
2446 const BlockByrefInfo &byrefInfo,
2447 BlockByrefHelpers &generator) {
2448 CodeGenFunction CGF(CGM);
2449 return generateByrefCopyHelper(CGF, byrefInfo, generator);
2450}
2451
2452/// Generate code for a __block variable's dispose helper.
2453static llvm::Constant *
2454generateByrefDisposeHelper(CodeGenFunction &CGF,
2455 const BlockByrefInfo &byrefInfo,
2456 BlockByrefHelpers &generator) {
2457 ASTContext &Context = CGF.getContext();
2458 QualType R = Context.VoidTy;
2459
2460 auto *Src = ImplicitParamDecl::Create(C&: CGF.getContext(), T: Context.VoidPtrTy,
2461 ParamKind: ImplicitParamKind::Other);
2462
2463 FunctionArgList args{Src};
2464 const CGFunctionInfo &FI =
2465 CGF.CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: R, args);
2466
2467 llvm::FunctionType *LTy = CGF.CGM.getTypes().GetFunctionType(Info: FI);
2468
2469 // FIXME: We'd like to put these into a mergable by content, with
2470 // internal linkage.
2471 llvm::Function *Fn =
2472 llvm::Function::Create(Ty: LTy, Linkage: llvm::GlobalValue::InternalLinkage,
2473 N: "__Block_byref_object_dispose_",
2474 M: &CGF.CGM.getModule());
2475
2476 CGF.CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI);
2477
2478 CGF.StartFunction(GD: GlobalDecl(), RetTy: R, Fn, FnInfo: FI, Args: args);
2479 // Create a scope with an artificial location for the body of this function.
2480 auto AL = ApplyDebugLocation::CreateArtificial(CGF);
2481
2482 if (generator.needsDispose()) {
2483 Address addr = CGF.GetAddrOfLocalVar(VD: Src);
2484 addr = Address(CGF.Builder.CreateLoad(Addr: addr), byrefInfo.Type,
2485 byrefInfo.ByrefAlignment);
2486 addr = CGF.emitBlockByrefAddress(baseAddr: addr, info: byrefInfo, followForward: false, name: "object");
2487
2488 generator.emitDispose(CGF, field: addr);
2489 }
2490
2491 CGF.FinishFunction();
2492
2493 return Fn;
2494}
2495
2496/// Build the dispose helper for a __block variable.
2497static llvm::Constant *buildByrefDisposeHelper(CodeGenModule &CGM,
2498 const BlockByrefInfo &byrefInfo,
2499 BlockByrefHelpers &generator) {
2500 CodeGenFunction CGF(CGM);
2501 return generateByrefDisposeHelper(CGF, byrefInfo, generator);
2502}
2503
2504/// Lazily build the copy and dispose helpers for a __block variable
2505/// with the given information.
2506template <class T>
2507static T *buildByrefHelpers(CodeGenModule &CGM, const BlockByrefInfo &byrefInfo,
2508 T &&generator) {
2509 llvm::FoldingSetNodeID id;
2510 generator.Profile(id);
2511
2512 void *insertPos;
2513 BlockByrefHelpers *node
2514 = CGM.ByrefHelpersCache.FindNodeOrInsertPos(ID: id, InsertPos&: insertPos);
2515 if (node) return static_cast<T*>(node);
2516
2517 generator.CopyHelper = buildByrefCopyHelper(CGM, byrefInfo, generator);
2518 generator.DisposeHelper = buildByrefDisposeHelper(CGM, byrefInfo, generator);
2519
2520 T *copy = new (CGM.getContext()) T(std::forward<T>(generator));
2521 CGM.ByrefHelpersCache.InsertNode(copy, insertPos);
2522 return copy;
2523}
2524
2525/// Build the copy and dispose helpers for the given __block variable
2526/// emission. Places the helpers in the global cache. Returns null
2527/// if no helpers are required.
2528BlockByrefHelpers *
2529CodeGenFunction::buildByrefHelpers(llvm::StructType &byrefType,
2530 const AutoVarEmission &emission) {
2531 const VarDecl &var = *emission.Variable;
2532 assert(var.isEscapingByref() &&
2533 "only escaping __block variables need byref helpers");
2534
2535 QualType type = var.getType();
2536
2537 auto &byrefInfo = getBlockByrefInfo(var: &var);
2538
2539 // The alignment we care about for the purposes of uniquing byref
2540 // helpers is the alignment of the actual byref value field.
2541 CharUnits valueAlignment =
2542 byrefInfo.ByrefAlignment.alignmentAtOffset(offset: byrefInfo.FieldOffset);
2543
2544 if (const CXXRecordDecl *record = type->getAsCXXRecordDecl()) {
2545 const Expr *copyExpr =
2546 CGM.getContext().getBlockVarCopyInit(VD: &var).getCopyExpr();
2547 if (!copyExpr && record->hasTrivialDestructor()) return nullptr;
2548
2549 return ::buildByrefHelpers(
2550 CGM, byrefInfo, generator: CXXByrefHelpers(valueAlignment, type, copyExpr));
2551 }
2552 if (type.hasAddressDiscriminatedPointerAuth()) {
2553 return ::buildByrefHelpers(
2554 CGM, byrefInfo, generator: AddressDiscriminatedByrefHelpers(valueAlignment, type));
2555 }
2556 // If type is a non-trivial C struct type that is non-trivial to
2557 // destructly move or destroy, build the copy and dispose helpers.
2558 if (type.isNonTrivialToPrimitiveDestructiveMove() == QualType::PCK_Struct ||
2559 type.isDestructedType() == QualType::DK_nontrivial_c_struct)
2560 return ::buildByrefHelpers(
2561 CGM, byrefInfo, generator: NonTrivialCStructByrefHelpers(valueAlignment, type));
2562
2563 // Otherwise, if we don't have a retainable type, there's nothing to do.
2564 // that the runtime does extra copies.
2565 if (!type->isObjCRetainableType()) return nullptr;
2566
2567 Qualifiers qs = type.getQualifiers();
2568
2569 // If we have lifetime, that dominates.
2570 if (Qualifiers::ObjCLifetime lifetime = qs.getObjCLifetime()) {
2571 switch (lifetime) {
2572 case Qualifiers::OCL_None: llvm_unreachable("impossible");
2573
2574 // These are just bits as far as the runtime is concerned.
2575 case Qualifiers::OCL_ExplicitNone:
2576 case Qualifiers::OCL_Autoreleasing:
2577 return nullptr;
2578
2579 // Tell the runtime that this is ARC __weak, called by the
2580 // byref routines.
2581 case Qualifiers::OCL_Weak:
2582 return ::buildByrefHelpers(CGM, byrefInfo,
2583 generator: ARCWeakByrefHelpers(valueAlignment));
2584
2585 // ARC __strong __block variables need to be retained.
2586 case Qualifiers::OCL_Strong:
2587 // Block pointers need to be copied, and there's no direct
2588 // transfer possible.
2589 if (type->isBlockPointerType()) {
2590 return ::buildByrefHelpers(CGM, byrefInfo,
2591 generator: ARCStrongBlockByrefHelpers(valueAlignment));
2592
2593 // Otherwise, we transfer ownership of the retain from the stack
2594 // to the heap.
2595 } else {
2596 return ::buildByrefHelpers(CGM, byrefInfo,
2597 generator: ARCStrongByrefHelpers(valueAlignment));
2598 }
2599 }
2600 llvm_unreachable("fell out of lifetime switch!");
2601 }
2602
2603 BlockFieldFlags flags;
2604 if (type->isBlockPointerType()) {
2605 flags |= BLOCK_FIELD_IS_BLOCK;
2606 } else if (CGM.getContext().isObjCNSObjectType(Ty: type) ||
2607 type->isObjCObjectPointerType()) {
2608 flags |= BLOCK_FIELD_IS_OBJECT;
2609 } else {
2610 return nullptr;
2611 }
2612
2613 if (type.isObjCGCWeak())
2614 flags |= BLOCK_FIELD_IS_WEAK;
2615
2616 return ::buildByrefHelpers(CGM, byrefInfo,
2617 generator: ObjectByrefHelpers(valueAlignment, flags));
2618}
2619
2620Address CodeGenFunction::emitBlockByrefAddress(Address baseAddr,
2621 const VarDecl *var,
2622 bool followForward) {
2623 auto &info = getBlockByrefInfo(var);
2624 return emitBlockByrefAddress(baseAddr, info, followForward, name: var->getName());
2625}
2626
2627Address CodeGenFunction::emitBlockByrefAddress(Address baseAddr,
2628 const BlockByrefInfo &info,
2629 bool followForward,
2630 const llvm::Twine &name) {
2631 // Chase the forwarding address if requested.
2632 if (followForward) {
2633 Address forwardingAddr = Builder.CreateStructGEP(Addr: baseAddr, Index: 1, Name: "forwarding");
2634 baseAddr = Address(Builder.CreateLoad(Addr: forwardingAddr), info.Type,
2635 info.ByrefAlignment);
2636 }
2637
2638 return Builder.CreateStructGEP(Addr: baseAddr, Index: info.FieldIndex, Name: name);
2639}
2640
2641/// BuildByrefInfo - This routine changes a __block variable declared as T x
2642/// into:
2643///
2644/// struct {
2645/// void *__isa;
2646/// void *__forwarding;
2647/// int32_t __flags;
2648/// int32_t __size;
2649/// void *__copy_helper; // only if needed
2650/// void *__destroy_helper; // only if needed
2651/// void *__byref_variable_layout;// only if needed
2652/// char padding[X]; // only if needed
2653/// T x;
2654/// } x
2655///
2656const BlockByrefInfo &CodeGenFunction::getBlockByrefInfo(const VarDecl *D) {
2657 auto it = BlockByrefInfos.find(Val: D);
2658 if (it != BlockByrefInfos.end())
2659 return it->second;
2660
2661 QualType Ty = D->getType();
2662
2663 CharUnits size;
2664 SmallVector<llvm::Type *, 8> types;
2665
2666 // void *__isa;
2667 types.push_back(Elt: VoidPtrTy);
2668 size += getPointerSize();
2669
2670 // void *__forwarding;
2671 types.push_back(Elt: VoidPtrTy);
2672 size += getPointerSize();
2673
2674 // int32_t __flags;
2675 types.push_back(Elt: Int32Ty);
2676 size += CharUnits::fromQuantity(Quantity: 4);
2677
2678 // int32_t __size;
2679 types.push_back(Elt: Int32Ty);
2680 size += CharUnits::fromQuantity(Quantity: 4);
2681
2682 // Note that this must match *exactly* the logic in buildByrefHelpers.
2683 bool hasCopyAndDispose = getContext().BlockRequiresCopying(Ty, D);
2684 if (hasCopyAndDispose) {
2685 /// void *__copy_helper;
2686 types.push_back(Elt: VoidPtrTy);
2687 size += getPointerSize();
2688
2689 /// void *__destroy_helper;
2690 types.push_back(Elt: VoidPtrTy);
2691 size += getPointerSize();
2692 }
2693
2694 bool HasByrefExtendedLayout = false;
2695 Qualifiers::ObjCLifetime Lifetime = Qualifiers::OCL_None;
2696 if (getContext().getByrefLifetime(Ty, Lifetime, HasByrefExtendedLayout) &&
2697 HasByrefExtendedLayout) {
2698 /// void *__byref_variable_layout;
2699 types.push_back(Elt: VoidPtrTy);
2700 size += CharUnits::fromQuantity(Quantity: PointerSizeInBytes);
2701 }
2702
2703 // T x;
2704 llvm::Type *varTy = ConvertTypeForMem(T: Ty);
2705
2706 bool packed = false;
2707 CharUnits varAlign = getContext().getDeclAlign(D);
2708 CharUnits varOffset = size.alignTo(Align: varAlign);
2709
2710 // We may have to insert padding.
2711 if (varOffset != size) {
2712 llvm::Type *paddingTy =
2713 llvm::ArrayType::get(ElementType: Int8Ty, NumElements: (varOffset - size).getQuantity());
2714
2715 types.push_back(Elt: paddingTy);
2716 size = varOffset;
2717
2718 // Conversely, we might have to prevent LLVM from inserting padding.
2719 } else if (CGM.getDataLayout().getABITypeAlign(Ty: varTy) >
2720 uint64_t(varAlign.getQuantity())) {
2721 packed = true;
2722 }
2723 types.push_back(Elt: varTy);
2724
2725 llvm::StructType *byrefType = llvm::StructType::create(
2726 Context&: getLLVMContext(), Elements: types, Name: "struct.__block_byref_" + D->getNameAsString(),
2727 isPacked: packed);
2728
2729 BlockByrefInfo info;
2730 info.Type = byrefType;
2731 info.FieldIndex = types.size() - 1;
2732 info.FieldOffset = varOffset;
2733 info.ByrefAlignment = std::max(a: varAlign, b: getPointerAlign());
2734
2735 auto pair = BlockByrefInfos.insert(KV: {D, info});
2736 assert(pair.second && "info was inserted recursively?");
2737 return pair.first->second;
2738}
2739
2740/// Initialize the structural components of a __block variable, i.e.
2741/// everything but the actual object.
2742void CodeGenFunction::emitByrefStructureInit(const AutoVarEmission &emission) {
2743 // Find the address of the local.
2744 Address addr = emission.Addr;
2745
2746 // That's an alloca of the byref structure type.
2747 llvm::StructType *byrefType = cast<llvm::StructType>(Val: addr.getElementType());
2748
2749 unsigned nextHeaderIndex = 0;
2750 CharUnits nextHeaderOffset;
2751 auto storeHeaderField = [&](llvm::Value *value, CharUnits fieldSize,
2752 const Twine &name, bool isFunction = false) {
2753 auto fieldAddr = Builder.CreateStructGEP(Addr: addr, Index: nextHeaderIndex, Name: name);
2754 if (isFunction) {
2755 if (auto &Schema = CGM.getCodeGenOpts()
2756 .PointerAuth.BlockByrefHelperFunctionPointers) {
2757 auto PointerAuth = EmitPointerAuthInfo(
2758 Schema, StorageAddress: fieldAddr.emitRawPointer(CGF&: *this), SchemaDecl: GlobalDecl(), SchemaType: QualType());
2759 value = EmitPointerAuthSign(Info: PointerAuth, Pointer: value);
2760 }
2761 }
2762 Builder.CreateStore(Val: value, Addr: fieldAddr);
2763
2764 nextHeaderIndex++;
2765 nextHeaderOffset += fieldSize;
2766 };
2767
2768 // Build the byref helpers if necessary. This is null if we don't need any.
2769 BlockByrefHelpers *helpers = buildByrefHelpers(byrefType&: *byrefType, emission);
2770
2771 const VarDecl &D = *emission.Variable;
2772 QualType type = D.getType();
2773
2774 bool HasByrefExtendedLayout = false;
2775 Qualifiers::ObjCLifetime ByrefLifetime = Qualifiers::OCL_None;
2776 bool ByRefHasLifetime =
2777 getContext().getByrefLifetime(Ty: type, Lifetime&: ByrefLifetime, HasByrefExtendedLayout);
2778
2779 llvm::Value *V;
2780
2781 // Initialize the 'isa', which is just 0 or 1.
2782 int isa = 0;
2783 if (type.isObjCGCWeak())
2784 isa = 1;
2785 V = Builder.CreateIntToPtr(V: Builder.getInt32(C: isa), DestTy: Int8PtrTy, Name: "isa");
2786 storeHeaderField(V, getPointerSize(), "byref.isa");
2787
2788 // Store the address of the variable into its own forwarding pointer.
2789 storeHeaderField(addr.emitRawPointer(CGF&: *this), getPointerSize(),
2790 "byref.forwarding");
2791
2792 // Blocks ABI:
2793 // c) the flags field is set to either 0 if no helper functions are
2794 // needed or BLOCK_BYREF_HAS_COPY_DISPOSE if they are,
2795 BlockFlags flags;
2796 if (helpers) flags |= BLOCK_BYREF_HAS_COPY_DISPOSE;
2797 if (ByRefHasLifetime) {
2798 if (HasByrefExtendedLayout) flags |= BLOCK_BYREF_LAYOUT_EXTENDED;
2799 else switch (ByrefLifetime) {
2800 case Qualifiers::OCL_Strong:
2801 flags |= BLOCK_BYREF_LAYOUT_STRONG;
2802 break;
2803 case Qualifiers::OCL_Weak:
2804 flags |= BLOCK_BYREF_LAYOUT_WEAK;
2805 break;
2806 case Qualifiers::OCL_ExplicitNone:
2807 flags |= BLOCK_BYREF_LAYOUT_UNRETAINED;
2808 break;
2809 case Qualifiers::OCL_None:
2810 if (!type->isObjCObjectPointerType() && !type->isBlockPointerType())
2811 flags |= BLOCK_BYREF_LAYOUT_NON_OBJECT;
2812 break;
2813 default:
2814 break;
2815 }
2816 if (CGM.getLangOpts().ObjCGCBitmapPrint) {
2817 printf(format: "\n Inline flag for BYREF variable layout (%d):", flags.getBitMask());
2818 if (flags & BLOCK_BYREF_HAS_COPY_DISPOSE)
2819 printf(format: " BLOCK_BYREF_HAS_COPY_DISPOSE");
2820 if (flags & BLOCK_BYREF_LAYOUT_MASK) {
2821 BlockFlags ThisFlag(flags.getBitMask() & BLOCK_BYREF_LAYOUT_MASK);
2822 if (ThisFlag == BLOCK_BYREF_LAYOUT_EXTENDED)
2823 printf(format: " BLOCK_BYREF_LAYOUT_EXTENDED");
2824 if (ThisFlag == BLOCK_BYREF_LAYOUT_STRONG)
2825 printf(format: " BLOCK_BYREF_LAYOUT_STRONG");
2826 if (ThisFlag == BLOCK_BYREF_LAYOUT_WEAK)
2827 printf(format: " BLOCK_BYREF_LAYOUT_WEAK");
2828 if (ThisFlag == BLOCK_BYREF_LAYOUT_UNRETAINED)
2829 printf(format: " BLOCK_BYREF_LAYOUT_UNRETAINED");
2830 if (ThisFlag == BLOCK_BYREF_LAYOUT_NON_OBJECT)
2831 printf(format: " BLOCK_BYREF_LAYOUT_NON_OBJECT");
2832 }
2833 printf(format: "\n");
2834 }
2835 }
2836 storeHeaderField(llvm::ConstantInt::get(Ty: IntTy, V: flags.getBitMask()),
2837 getIntSize(), "byref.flags");
2838
2839 CharUnits byrefSize = CGM.GetTargetTypeStoreSize(Ty: byrefType);
2840 V = llvm::ConstantInt::get(Ty: IntTy, V: byrefSize.getQuantity());
2841 storeHeaderField(V, getIntSize(), "byref.size");
2842
2843 if (helpers) {
2844 storeHeaderField(helpers->CopyHelper, getPointerSize(), "byref.copyHelper",
2845 /*isFunction=*/true);
2846 storeHeaderField(helpers->DisposeHelper, getPointerSize(),
2847 "byref.disposeHelper", /*isFunction=*/true);
2848 }
2849
2850 if (ByRefHasLifetime && HasByrefExtendedLayout) {
2851 auto layoutInfo = CGM.getObjCRuntime().BuildByrefLayout(CGM, T: type);
2852 storeHeaderField(layoutInfo, getPointerSize(), "byref.layout");
2853 }
2854}
2855
2856void CodeGenFunction::BuildBlockRelease(llvm::Value *V, BlockFieldFlags flags,
2857 bool CanThrow) {
2858 llvm::FunctionCallee F = CGM.getBlockObjectDispose();
2859 llvm::Value *args[] = {V,
2860 llvm::ConstantInt::get(Ty: Int32Ty, V: flags.getBitMask())};
2861
2862 if (CanThrow)
2863 EmitRuntimeCallOrInvoke(callee: F, args);
2864 else
2865 EmitNounwindRuntimeCall(callee: F, args);
2866}
2867
2868void CodeGenFunction::enterByrefCleanup(CleanupKind Kind, Address Addr,
2869 BlockFieldFlags Flags,
2870 bool LoadBlockVarAddr, bool CanThrow) {
2871 EHStack.pushCleanup<CallBlockRelease>(Kind, A: Addr, A: Flags, A: LoadBlockVarAddr,
2872 A: CanThrow);
2873}
2874
2875/// Adjust the declaration of something from the blocks API.
2876static void configureBlocksRuntimeObject(CodeGenModule &CGM,
2877 llvm::Constant *C) {
2878 auto *GV = cast<llvm::GlobalValue>(Val: C->stripPointerCasts());
2879
2880 if (!CGM.getCodeGenOpts().StaticClosure &&
2881 CGM.getTarget().getTriple().isOSBinFormatCOFF()) {
2882 const IdentifierInfo &II = CGM.getContext().Idents.get(Name: C->getName());
2883 TranslationUnitDecl *TUDecl = CGM.getContext().getTranslationUnitDecl();
2884 DeclContext *DC = TranslationUnitDecl::castToDeclContext(D: TUDecl);
2885
2886 assert((isa<llvm::Function>(C->stripPointerCasts()) ||
2887 isa<llvm::GlobalVariable>(C->stripPointerCasts())) &&
2888 "expected Function or GlobalVariable");
2889
2890 const NamedDecl *ND = nullptr;
2891 for (const auto *Result : DC->lookup(Name: &II))
2892 if ((ND = dyn_cast<FunctionDecl>(Val: Result)) ||
2893 (ND = dyn_cast<VarDecl>(Val: Result)))
2894 break;
2895
2896 if (GV->isDeclaration() && (!ND || !ND->hasAttr<DLLExportAttr>())) {
2897 GV->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass);
2898 GV->setLinkage(llvm::GlobalValue::ExternalLinkage);
2899 } else {
2900 GV->setDLLStorageClass(llvm::GlobalValue::DLLExportStorageClass);
2901 GV->setLinkage(llvm::GlobalValue::ExternalLinkage);
2902 }
2903 }
2904
2905 if (CGM.getLangOpts().BlocksRuntimeOptional && GV->isDeclaration() &&
2906 GV->hasExternalLinkage())
2907 GV->setLinkage(llvm::GlobalValue::ExternalWeakLinkage);
2908
2909 CGM.setDSOLocal(GV);
2910}
2911
2912llvm::FunctionCallee CodeGenModule::getBlockObjectDispose() {
2913 if (BlockObjectDispose)
2914 return BlockObjectDispose;
2915
2916 QualType args[] = {Context.VoidPtrTy, Context.IntTy};
2917 BlockObjectDispose =
2918 CreateRuntimeFunction(ReturnTy: Context.VoidTy, ArgTys: args, Name: "_Block_object_dispose");
2919 configureBlocksRuntimeObject(
2920 CGM&: *this, C: cast<llvm::Constant>(Val: BlockObjectDispose.getCallee()));
2921 return BlockObjectDispose;
2922}
2923
2924llvm::FunctionCallee CodeGenModule::getBlockObjectAssign() {
2925 if (BlockObjectAssign)
2926 return BlockObjectAssign;
2927
2928 QualType args[] = {Context.VoidPtrTy, Context.VoidPtrTy, Context.IntTy};
2929 BlockObjectAssign =
2930 CreateRuntimeFunction(ReturnTy: Context.VoidTy, ArgTys: args, Name: "_Block_object_assign");
2931 configureBlocksRuntimeObject(
2932 CGM&: *this, C: cast<llvm::Constant>(Val: BlockObjectAssign.getCallee()));
2933 return BlockObjectAssign;
2934}
2935
2936llvm::Constant *CodeGenModule::getNSConcreteGlobalBlock() {
2937 if (NSConcreteGlobalBlock)
2938 return NSConcreteGlobalBlock;
2939
2940 NSConcreteGlobalBlock = GetOrCreateLLVMGlobal(
2941 MangledName: "_NSConcreteGlobalBlock", Ty: Int8PtrTy, AddrSpace: LangAS::Default, D: nullptr);
2942 configureBlocksRuntimeObject(CGM&: *this, C: NSConcreteGlobalBlock);
2943 return NSConcreteGlobalBlock;
2944}
2945
2946llvm::Constant *CodeGenModule::getNSConcreteStackBlock() {
2947 if (NSConcreteStackBlock)
2948 return NSConcreteStackBlock;
2949
2950 NSConcreteStackBlock = GetOrCreateLLVMGlobal(
2951 MangledName: "_NSConcreteStackBlock", Ty: Int8PtrTy, AddrSpace: LangAS::Default, D: nullptr);
2952 configureBlocksRuntimeObject(CGM&: *this, C: NSConcreteStackBlock);
2953 return NSConcreteStackBlock;
2954}
2955