1//===--- CGBlocks.cpp - Emit LLVM Code for declarations ---------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit blocks.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGBlocks.h"
14#include "CGCXXABI.h"
15#include "CGDebugInfo.h"
16#include "CGObjCRuntime.h"
17#include "CGOpenCLRuntime.h"
18#include "CodeGenFunction.h"
19#include "CodeGenModule.h"
20#include "CodeGenPGO.h"
21#include "ConstantEmitter.h"
22#include "TargetInfo.h"
23#include "clang/AST/Attr.h"
24#include "clang/AST/DeclObjC.h"
25#include "clang/CodeGen/ConstantInitBuilder.h"
26#include "llvm/IR/DataLayout.h"
27#include "llvm/IR/Module.h"
28#include "llvm/Support/ScopedPrinter.h"
29#include <algorithm>
30#include <cstdio>
31
32using namespace clang;
33using namespace CodeGen;
34
35CGBlockInfo::CGBlockInfo(const BlockDecl *block, StringRef name)
36 : Name(name), CXXThisIndex(0), CanBeGlobal(false), NeedsCopyDispose(false),
37 NoEscape(false), HasCXXObject(false), UsesStret(false),
38 HasCapturedVariableLayout(false), CapturesNonExternalType(false),
39 LocalAddress(RawAddress::invalid()), StructureType(nullptr),
40 Block(block) {
41
42 // Skip asm prefix, if any. 'name' is usually taken directly from
43 // the mangled name of the enclosing function.
44 name.consume_front(Prefix: "\01");
45}
46
47// Anchor the vtable to this translation unit.
48BlockByrefHelpers::~BlockByrefHelpers() {}
49
50/// Build the given block as a global block.
51static llvm::Constant *buildGlobalBlock(CodeGenModule &CGM,
52 const CGBlockInfo &blockInfo,
53 llvm::Constant *blockFn);
54
55/// Build the helper function to copy a block.
56static llvm::Constant *buildCopyHelper(CodeGenModule &CGM,
57 const CGBlockInfo &blockInfo) {
58 return CodeGenFunction(CGM).GenerateCopyHelperFunction(blockInfo);
59}
60
61/// Build the helper function to dispose of a block.
62static llvm::Constant *buildDisposeHelper(CodeGenModule &CGM,
63 const CGBlockInfo &blockInfo) {
64 return CodeGenFunction(CGM).GenerateDestroyHelperFunction(blockInfo);
65}
66
67namespace {
68
69enum class CaptureStrKind {
70 // String for the copy helper.
71 CopyHelper,
72 // String for the dispose helper.
73 DisposeHelper,
74 // Merge the strings for the copy helper and dispose helper.
75 Merged
76};
77
78} // end anonymous namespace
79
80static std::string getBlockCaptureStr(const CGBlockInfo::Capture &Cap,
81 CaptureStrKind StrKind,
82 CharUnits BlockAlignment,
83 CodeGenModule &CGM);
84
85static std::string getBlockDescriptorName(const CGBlockInfo &BlockInfo,
86 CodeGenModule &CGM) {
87 std::string Name = "__block_descriptor_";
88 Name += llvm::to_string(Value: BlockInfo.BlockSize.getQuantity()) + "_";
89
90 if (BlockInfo.NeedsCopyDispose) {
91 if (CGM.getLangOpts().Exceptions)
92 Name += "e";
93 if (CGM.getCodeGenOpts().ObjCAutoRefCountExceptions)
94 Name += "a";
95 Name += llvm::to_string(Value: BlockInfo.BlockAlign.getQuantity()) + "_";
96
97 for (auto &Cap : BlockInfo.SortedCaptures) {
98 if (Cap.isConstantOrTrivial())
99 continue;
100
101 Name += llvm::to_string(Value: Cap.getOffset().getQuantity());
102
103 if (Cap.CopyKind == Cap.DisposeKind) {
104 // If CopyKind and DisposeKind are the same, merge the capture
105 // information.
106 assert(Cap.CopyKind != BlockCaptureEntityKind::None &&
107 "shouldn't see BlockCaptureManagedEntity that is None");
108 Name += getBlockCaptureStr(Cap, StrKind: CaptureStrKind::Merged,
109 BlockAlignment: BlockInfo.BlockAlign, CGM);
110 } else {
111 // If CopyKind and DisposeKind are not the same, which can happen when
112 // either Kind is None or the captured object is a __strong block,
113 // concatenate the copy and dispose strings.
114 Name += getBlockCaptureStr(Cap, StrKind: CaptureStrKind::CopyHelper,
115 BlockAlignment: BlockInfo.BlockAlign, CGM);
116 Name += getBlockCaptureStr(Cap, StrKind: CaptureStrKind::DisposeHelper,
117 BlockAlignment: BlockInfo.BlockAlign, CGM);
118 }
119 }
120 Name += "_";
121 }
122
123 std::string TypeAtEncoding;
124
125 if (!CGM.getCodeGenOpts().DisableBlockSignatureString) {
126 TypeAtEncoding =
127 CGM.getContext().getObjCEncodingForBlock(blockExpr: BlockInfo.getBlockExpr());
128 /// Replace occurrences of '@' with '\1'. '@' is reserved on ELF platforms
129 /// as a separator between symbol name and symbol version.
130 llvm::replace(Range&: TypeAtEncoding, OldValue: '@', NewValue: '\1');
131 }
132 Name += "e" + llvm::to_string(Value: TypeAtEncoding.size()) + "_" + TypeAtEncoding;
133 Name += "l" + CGM.getObjCRuntime().getRCBlockLayoutStr(CGM, blockInfo: BlockInfo);
134 return Name;
135}
136
137/// buildBlockDescriptor - Build the block descriptor meta-data for a block.
138/// buildBlockDescriptor is accessed from 5th field of the Block_literal
139/// meta-data and contains stationary information about the block literal.
140/// Its definition will have 4 (or optionally 6) words.
141/// \code
142/// struct Block_descriptor {
143/// unsigned long reserved;
144/// unsigned long size; // size of Block_literal metadata in bytes.
145/// void *copy_func_helper_decl; // optional copy helper.
146/// void *destroy_func_decl; // optional destructor helper.
147/// void *block_method_encoding_address; // @encode for block literal signature.
148/// void *block_layout_info; // encoding of captured block variables.
149/// };
150/// \endcode
151static llvm::Constant *buildBlockDescriptor(CodeGenModule &CGM,
152 const CGBlockInfo &blockInfo) {
153 ASTContext &C = CGM.getContext();
154
155 llvm::IntegerType *ulong =
156 cast<llvm::IntegerType>(Val: CGM.getTypes().ConvertType(T: C.UnsignedLongTy));
157 llvm::PointerType *i8p = nullptr;
158 if (CGM.getLangOpts().OpenCL)
159 i8p = llvm::PointerType::get(
160 C&: CGM.getLLVMContext(), AddressSpace: C.getTargetAddressSpace(AS: LangAS::opencl_constant));
161 else
162 i8p = CGM.VoidPtrTy;
163
164 std::string descName;
165
166 // If an equivalent block descriptor global variable exists, return it.
167 if (C.getLangOpts().ObjC &&
168 CGM.getLangOpts().getGC() == LangOptions::NonGC) {
169 descName = getBlockDescriptorName(BlockInfo: blockInfo, CGM);
170 if (llvm::GlobalValue *desc = CGM.getModule().getNamedValue(Name: descName))
171 return desc;
172 }
173
174 // If there isn't an equivalent block descriptor global variable, create a new
175 // one.
176 ConstantInitBuilder builder(CGM);
177 auto elements = builder.beginStruct();
178
179 // reserved
180 elements.addInt(intTy: ulong, value: 0);
181
182 // Size
183 // FIXME: What is the right way to say this doesn't fit? We should give
184 // a user diagnostic in that case. Better fix would be to change the
185 // API to size_t.
186 elements.addInt(intTy: ulong, value: blockInfo.BlockSize.getQuantity());
187
188 // Optional copy/dispose helpers.
189 bool hasInternalHelper = false;
190 if (blockInfo.NeedsCopyDispose) {
191 auto &Schema = CGM.getCodeGenOpts().PointerAuth.BlockHelperFunctionPointers;
192 // copy_func_helper_decl
193 llvm::Constant *copyHelper = buildCopyHelper(CGM, blockInfo);
194 elements.addSignedPointer(Pointer: copyHelper, Schema, CalleeDecl: GlobalDecl(), CalleeType: QualType());
195
196 // destroy_func_decl
197 llvm::Constant *disposeHelper = buildDisposeHelper(CGM, blockInfo);
198 elements.addSignedPointer(Pointer: disposeHelper, Schema, CalleeDecl: GlobalDecl(), CalleeType: QualType());
199
200 if (cast<llvm::Function>(Val: copyHelper->stripPointerCasts())
201 ->hasInternalLinkage() ||
202 cast<llvm::Function>(Val: disposeHelper->stripPointerCasts())
203 ->hasInternalLinkage())
204 hasInternalHelper = true;
205 }
206
207 // Signature. Mandatory ObjC-style method descriptor @encode sequence.
208 if (CGM.getCodeGenOpts().DisableBlockSignatureString) {
209 elements.addNullPointer(ptrTy: i8p);
210 } else {
211 std::string typeAtEncoding =
212 CGM.getContext().getObjCEncodingForBlock(blockExpr: blockInfo.getBlockExpr());
213 elements.add(value: CGM.GetAddrOfConstantCString(Str: typeAtEncoding).getPointer());
214 }
215
216 // GC layout.
217 if (C.getLangOpts().ObjC) {
218 if (CGM.getLangOpts().getGC() != LangOptions::NonGC)
219 elements.add(value: CGM.getObjCRuntime().BuildGCBlockLayout(CGM, blockInfo));
220 else
221 elements.add(value: CGM.getObjCRuntime().BuildRCBlockLayout(CGM, blockInfo));
222 }
223 else
224 elements.addNullPointer(ptrTy: i8p);
225
226 unsigned AddrSpace = 0;
227 if (C.getLangOpts().OpenCL)
228 AddrSpace = C.getTargetAddressSpace(AS: LangAS::opencl_constant);
229
230 llvm::GlobalValue::LinkageTypes linkage;
231 if (descName.empty()) {
232 linkage = llvm::GlobalValue::InternalLinkage;
233 descName = "__block_descriptor_tmp";
234 } else if (hasInternalHelper) {
235 // If either the copy helper or the dispose helper has internal linkage,
236 // the block descriptor must have internal linkage too.
237 linkage = llvm::GlobalValue::InternalLinkage;
238 } else {
239 linkage = llvm::GlobalValue::LinkOnceODRLinkage;
240 }
241
242 llvm::GlobalVariable *global =
243 elements.finishAndCreateGlobal(args&: descName, args: CGM.getPointerAlign(),
244 /*constant*/ args: true, args&: linkage, args&: AddrSpace);
245
246 if (linkage == llvm::GlobalValue::LinkOnceODRLinkage) {
247 if (CGM.supportsCOMDAT())
248 global->setComdat(CGM.getModule().getOrInsertComdat(Name: descName));
249 global->setVisibility(llvm::GlobalValue::HiddenVisibility);
250 global->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
251 }
252
253 return global;
254}
255
256/*
257 Purely notional variadic template describing the layout of a block.
258
259 template <class _ResultType, class... _ParamTypes, class... _CaptureTypes>
260 struct Block_literal {
261 /// Initialized to one of:
262 /// extern void *_NSConcreteStackBlock[];
263 /// extern void *_NSConcreteGlobalBlock[];
264 ///
265 /// In theory, we could start one off malloc'ed by setting
266 /// BLOCK_NEEDS_FREE, giving it a refcount of 1, and using
267 /// this isa:
268 /// extern void *_NSConcreteMallocBlock[];
269 struct objc_class *isa;
270
271 /// These are the flags (with corresponding bit number) that the
272 /// compiler is actually supposed to know about.
273 /// 23. BLOCK_IS_NOESCAPE - indicates that the block is non-escaping
274 /// 25. BLOCK_HAS_COPY_DISPOSE - indicates that the block
275 /// descriptor provides copy and dispose helper functions
276 /// 26. BLOCK_HAS_CXX_OBJ - indicates that there's a captured
277 /// object with a nontrivial destructor or copy constructor
278 /// 28. BLOCK_IS_GLOBAL - indicates that the block is allocated
279 /// as global memory
280 /// 29. BLOCK_USE_STRET - indicates that the block function
281 /// uses stret, which objc_msgSend needs to know about
282 /// 30. BLOCK_HAS_SIGNATURE - indicates that the block has an
283 /// @encoded signature string
284 /// And we're not supposed to manipulate these:
285 /// 24. BLOCK_NEEDS_FREE - indicates that the block has been moved
286 /// to malloc'ed memory
287 /// 27. BLOCK_IS_GC - indicates that the block has been moved to
288 /// to GC-allocated memory
289 /// Additionally, the bottom 16 bits are a reference count which
290 /// should be zero on the stack.
291 int flags;
292
293 /// Reserved; should be zero-initialized.
294 int reserved;
295
296 /// Function pointer generated from block literal.
297 _ResultType (*invoke)(Block_literal *, _ParamTypes...);
298
299 /// Block description metadata generated from block literal.
300 struct Block_descriptor *block_descriptor;
301
302 /// Captured values follow.
303 _CapturesTypes captures...;
304 };
305 */
306
307namespace {
308 /// A chunk of data that we actually have to capture in the block.
309 struct BlockLayoutChunk {
310 CharUnits Alignment;
311 CharUnits Size;
312 const BlockDecl::Capture *Capture; // null for 'this'
313 llvm::Type *Type;
314 QualType FieldType;
315 BlockCaptureEntityKind CopyKind, DisposeKind;
316 BlockFieldFlags CopyFlags, DisposeFlags;
317
318 BlockLayoutChunk(CharUnits align, CharUnits size,
319 const BlockDecl::Capture *capture, llvm::Type *type,
320 QualType fieldType, BlockCaptureEntityKind CopyKind,
321 BlockFieldFlags CopyFlags,
322 BlockCaptureEntityKind DisposeKind,
323 BlockFieldFlags DisposeFlags)
324 : Alignment(align), Size(size), Capture(capture), Type(type),
325 FieldType(fieldType), CopyKind(CopyKind), DisposeKind(DisposeKind),
326 CopyFlags(CopyFlags), DisposeFlags(DisposeFlags) {}
327
328 /// Tell the block info that this chunk has the given field index.
329 void setIndex(CGBlockInfo &info, unsigned index, CharUnits offset) {
330 if (!Capture) {
331 info.CXXThisIndex = index;
332 info.CXXThisOffset = offset;
333 } else {
334 info.SortedCaptures.push_back(Elt: CGBlockInfo::Capture::makeIndex(
335 index, offset, FieldType, CopyKind, CopyFlags, DisposeKind,
336 DisposeFlags, Cap: Capture));
337 }
338 }
339
340 bool isTrivial() const {
341 return CopyKind == BlockCaptureEntityKind::None &&
342 DisposeKind == BlockCaptureEntityKind::None;
343 }
344 };
345
346 /// Order by 1) all __strong together 2) next, all block together 3) next,
347 /// all byref together 4) next, all __weak together. Preserve descending
348 /// alignment in all situations.
349 bool operator<(const BlockLayoutChunk &left, const BlockLayoutChunk &right) {
350 if (left.Alignment != right.Alignment)
351 return left.Alignment > right.Alignment;
352
353 auto getPrefOrder = [](const BlockLayoutChunk &chunk) {
354 switch (chunk.CopyKind) {
355 case BlockCaptureEntityKind::ARCStrong:
356 return 0;
357 case BlockCaptureEntityKind::BlockObject:
358 switch (chunk.CopyFlags.getBitMask()) {
359 case BLOCK_FIELD_IS_OBJECT:
360 return 0;
361 case BLOCK_FIELD_IS_BLOCK:
362 return 1;
363 case BLOCK_FIELD_IS_BYREF:
364 return 2;
365 default:
366 break;
367 }
368 break;
369 case BlockCaptureEntityKind::ARCWeak:
370 return 3;
371 default:
372 break;
373 }
374 return 4;
375 };
376
377 return getPrefOrder(left) < getPrefOrder(right);
378 }
379} // end anonymous namespace
380
381static std::pair<BlockCaptureEntityKind, BlockFieldFlags>
382computeCopyInfoForBlockCapture(const BlockDecl::Capture &CI, QualType T,
383 const LangOptions &LangOpts);
384
385static std::pair<BlockCaptureEntityKind, BlockFieldFlags>
386computeDestroyInfoForBlockCapture(const BlockDecl::Capture &CI, QualType T,
387 const LangOptions &LangOpts);
388
389static void addBlockLayout(CharUnits align, CharUnits size,
390 const BlockDecl::Capture *capture, llvm::Type *type,
391 QualType fieldType,
392 SmallVectorImpl<BlockLayoutChunk> &Layout,
393 CGBlockInfo &Info, CodeGenModule &CGM) {
394 if (!capture) {
395 // 'this' capture.
396 Layout.push_back(Elt: BlockLayoutChunk(
397 align, size, capture, type, fieldType, BlockCaptureEntityKind::None,
398 BlockFieldFlags(), BlockCaptureEntityKind::None, BlockFieldFlags()));
399 return;
400 }
401
402 const LangOptions &LangOpts = CGM.getLangOpts();
403 BlockCaptureEntityKind CopyKind, DisposeKind;
404 BlockFieldFlags CopyFlags, DisposeFlags;
405
406 std::tie(args&: CopyKind, args&: CopyFlags) =
407 computeCopyInfoForBlockCapture(CI: *capture, T: fieldType, LangOpts);
408 std::tie(args&: DisposeKind, args&: DisposeFlags) =
409 computeDestroyInfoForBlockCapture(CI: *capture, T: fieldType, LangOpts);
410 Layout.push_back(Elt: BlockLayoutChunk(align, size, capture, type, fieldType,
411 CopyKind, CopyFlags, DisposeKind,
412 DisposeFlags));
413
414 if (Info.NoEscape)
415 return;
416
417 if (!Layout.back().isTrivial())
418 Info.NeedsCopyDispose = true;
419}
420
421/// Determines if the given type is safe for constant capture in C++.
422static bool isSafeForCXXConstantCapture(QualType type) {
423 const auto *record = type->getBaseElementTypeUnsafe()->getAsCXXRecordDecl();
424
425 // Only records can be unsafe.
426 if (!record)
427 return true;
428
429 // Maintain semantics for classes with non-trivial dtors or copy ctors.
430 if (!record->hasTrivialDestructor()) return false;
431 if (record->hasNonTrivialCopyConstructor()) return false;
432
433 // Otherwise, we just have to make sure there aren't any mutable
434 // fields that might have changed since initialization.
435 return !record->hasMutableFields();
436}
437
438/// It is illegal to modify a const object after initialization.
439/// Therefore, if a const object has a constant initializer, we don't
440/// actually need to keep storage for it in the block; we'll just
441/// rematerialize it at the start of the block function. This is
442/// acceptable because we make no promises about address stability of
443/// captured variables.
444static llvm::Constant *tryCaptureAsConstant(CodeGenModule &CGM,
445 CodeGenFunction *CGF,
446 const VarDecl *var) {
447 // Return if this is a function parameter. We shouldn't try to
448 // rematerialize default arguments of function parameters.
449 if (isa<ParmVarDecl>(Val: var))
450 return nullptr;
451
452 QualType type = var->getType();
453
454 // We can only do this if the variable is const.
455 if (!type.isConstQualified()) return nullptr;
456
457 // Furthermore, in C++ we have to worry about mutable fields:
458 // C++ [dcl.type.cv]p4:
459 // Except that any class member declared mutable can be
460 // modified, any attempt to modify a const object during its
461 // lifetime results in undefined behavior.
462 if (CGM.getLangOpts().CPlusPlus && !isSafeForCXXConstantCapture(type))
463 return nullptr;
464
465 // If the variable doesn't have any initializer (shouldn't this be
466 // invalid?), it's not clear what we should do. Maybe capture as
467 // zero?
468 const Expr *init = var->getInit();
469 if (!init) return nullptr;
470
471 return ConstantEmitter(CGM, CGF).tryEmitAbstractForInitializer(D: *var);
472}
473
474/// Get the low bit of a nonzero character count. This is the
475/// alignment of the nth byte if the 0th byte is universally aligned.
476static CharUnits getLowBit(CharUnits v) {
477 return CharUnits::fromQuantity(Quantity: v.getQuantity() & (~v.getQuantity() + 1));
478}
479
480static void initializeForBlockHeader(CodeGenModule &CGM, CGBlockInfo &info,
481 SmallVectorImpl<llvm::Type*> &elementTypes) {
482
483 assert(elementTypes.empty());
484 if (CGM.getLangOpts().OpenCL) {
485 // The header is basically 'struct { int; int; generic void *;
486 // custom_fields; }'. Assert that struct is packed.
487 auto GenPtrAlign = CharUnits::fromQuantity(
488 Quantity: CGM.getTarget().getPointerAlign(AddrSpace: LangAS::opencl_generic) / 8);
489 auto GenPtrSize = CharUnits::fromQuantity(
490 Quantity: CGM.getTarget().getPointerWidth(AddrSpace: LangAS::opencl_generic) / 8);
491 assert(CGM.getIntSize() <= GenPtrSize);
492 assert(CGM.getIntAlign() <= GenPtrAlign);
493 assert((2 * CGM.getIntSize()).isMultipleOf(GenPtrAlign));
494 elementTypes.push_back(Elt: CGM.IntTy); /* total size */
495 elementTypes.push_back(Elt: CGM.IntTy); /* align */
496 elementTypes.push_back(
497 Elt: CGM.getOpenCLRuntime()
498 .getGenericVoidPointerType()); /* invoke function */
499 unsigned Offset =
500 2 * CGM.getIntSize().getQuantity() + GenPtrSize.getQuantity();
501 unsigned BlockAlign = GenPtrAlign.getQuantity();
502 if (auto *Helper =
503 CGM.getTargetCodeGenInfo().getTargetOpenCLBlockHelper()) {
504 for (auto *I : Helper->getCustomFieldTypes()) /* custom fields */ {
505 // TargetOpenCLBlockHelp needs to make sure the struct is packed.
506 // If necessary, add padding fields to the custom fields.
507 unsigned Align = CGM.getDataLayout().getABITypeAlign(Ty: I).value();
508 if (BlockAlign < Align)
509 BlockAlign = Align;
510 assert(Offset % Align == 0);
511 Offset += CGM.getDataLayout().getTypeAllocSize(Ty: I);
512 elementTypes.push_back(Elt: I);
513 }
514 }
515 info.BlockAlign = CharUnits::fromQuantity(Quantity: BlockAlign);
516 info.BlockSize = CharUnits::fromQuantity(Quantity: Offset);
517 } else {
518 // The header is basically 'struct { void *; int; int; void *; void *; }'.
519 // Assert that the struct is packed.
520 assert(CGM.getIntSize() <= CGM.getPointerSize());
521 assert(CGM.getIntAlign() <= CGM.getPointerAlign());
522 assert((2 * CGM.getIntSize()).isMultipleOf(CGM.getPointerAlign()));
523 info.BlockAlign = CGM.getPointerAlign();
524 info.BlockSize = 3 * CGM.getPointerSize() + 2 * CGM.getIntSize();
525 elementTypes.push_back(Elt: CGM.VoidPtrTy);
526 elementTypes.push_back(Elt: CGM.IntTy);
527 elementTypes.push_back(Elt: CGM.IntTy);
528 elementTypes.push_back(Elt: CGM.VoidPtrTy);
529 elementTypes.push_back(Elt: CGM.getBlockDescriptorType());
530 }
531}
532
533static QualType getCaptureFieldType(const CodeGenFunction &CGF,
534 const BlockDecl::Capture &CI) {
535 const VarDecl *VD = CI.getVariable();
536
537 // If the variable is captured by an enclosing block or lambda expression,
538 // use the type of the capture field.
539 if (CGF.BlockInfo && CI.isNested())
540 return CGF.BlockInfo->getCapture(var: VD).fieldType();
541 if (auto *FD = CGF.LambdaCaptureFields.lookup(Val: VD))
542 return FD->getType();
543 // If the captured variable is a non-escaping __block variable, the field
544 // type is the reference type. If the variable is a __block variable that
545 // already has a reference type, the field type is the variable's type.
546 return VD->isNonEscapingByref() ?
547 CGF.getContext().getLValueReferenceType(T: VD->getType()) : VD->getType();
548}
549
550/// Compute the layout of the given block. Attempts to lay the block
551/// out with minimal space requirements.
552static void computeBlockInfo(CodeGenModule &CGM, CodeGenFunction *CGF,
553 CGBlockInfo &info) {
554 ASTContext &C = CGM.getContext();
555 const BlockDecl *block = info.getBlockDecl();
556
557 SmallVector<llvm::Type*, 8> elementTypes;
558 initializeForBlockHeader(CGM, info, elementTypes);
559 bool hasNonConstantCustomFields = false;
560 if (auto *OpenCLHelper =
561 CGM.getTargetCodeGenInfo().getTargetOpenCLBlockHelper())
562 hasNonConstantCustomFields =
563 !OpenCLHelper->areAllCustomFieldValuesConstant(Info: info);
564 if (!block->hasCaptures() && !hasNonConstantCustomFields) {
565 info.StructureType =
566 llvm::StructType::get(Context&: CGM.getLLVMContext(), Elements: elementTypes, isPacked: true);
567 info.CanBeGlobal = true;
568 return;
569 } else if (C.getLangOpts().ObjC &&
570 CGM.getLangOpts().getGC() == LangOptions::NonGC)
571 info.HasCapturedVariableLayout = true;
572
573 if (block->doesNotEscape())
574 info.NoEscape = true;
575
576 // Collect the layout chunks.
577 SmallVector<BlockLayoutChunk, 16> layout;
578 layout.reserve(N: block->capturesCXXThis() +
579 (block->capture_end() - block->capture_begin()));
580
581 CharUnits maxFieldAlign;
582
583 // First, 'this'.
584 if (block->capturesCXXThis()) {
585 assert(CGF && isa_and_nonnull<CXXMethodDecl>(CGF->CurFuncDecl) &&
586 "Can't capture 'this' outside a method");
587 QualType thisType = cast<CXXMethodDecl>(Val: CGF->CurFuncDecl)->getThisType();
588
589 // Theoretically, this could be in a different address space, so
590 // don't assume standard pointer size/align.
591 llvm::Type *llvmType = CGM.getTypes().ConvertType(T: thisType);
592 auto TInfo = CGM.getContext().getTypeInfoInChars(T: thisType);
593 maxFieldAlign = std::max(a: maxFieldAlign, b: TInfo.Align);
594
595 addBlockLayout(align: TInfo.Align, size: TInfo.Width, capture: nullptr, type: llvmType, fieldType: thisType,
596 Layout&: layout, Info&: info, CGM);
597 }
598
599 // Next, all the block captures.
600 for (const auto &CI : block->captures()) {
601 const VarDecl *variable = CI.getVariable();
602
603 if (CI.isEscapingByref()) {
604 // Just use void* instead of a pointer to the byref type.
605 CharUnits align = CGM.getPointerAlign();
606 maxFieldAlign = std::max(a: maxFieldAlign, b: align);
607
608 // Since a __block variable cannot be captured by lambdas, its type and
609 // the capture field type should always match.
610 assert(CGF && getCaptureFieldType(*CGF, CI) == variable->getType() &&
611 "capture type differs from the variable type");
612 addBlockLayout(align, size: CGM.getPointerSize(), capture: &CI, type: CGM.VoidPtrTy,
613 fieldType: variable->getType(), Layout&: layout, Info&: info, CGM);
614 continue;
615 }
616
617 // Otherwise, build a layout chunk with the size and alignment of
618 // the declaration.
619 if (llvm::Constant *constant = tryCaptureAsConstant(CGM, CGF, var: variable)) {
620 info.SortedCaptures.push_back(
621 Elt: CGBlockInfo::Capture::makeConstant(value: constant, Cap: &CI));
622 continue;
623 }
624
625 QualType VT = getCaptureFieldType(CGF: *CGF, CI);
626
627 if (CGM.getLangOpts().CPlusPlus)
628 if (const CXXRecordDecl *record = VT->getAsCXXRecordDecl())
629 if (CI.hasCopyExpr() || !record->hasTrivialDestructor()) {
630 info.HasCXXObject = true;
631 if (!record->isExternallyVisible())
632 info.CapturesNonExternalType = true;
633 }
634
635 CharUnits size = C.getTypeSizeInChars(T: VT);
636 CharUnits align = C.getDeclAlign(D: variable);
637
638 maxFieldAlign = std::max(a: maxFieldAlign, b: align);
639
640 llvm::Type *llvmType =
641 CGM.getTypes().ConvertTypeForMem(T: VT);
642
643 addBlockLayout(align, size, capture: &CI, type: llvmType, fieldType: VT, Layout&: layout, Info&: info, CGM);
644 }
645
646 // If that was everything, we're done here.
647 if (layout.empty()) {
648 info.StructureType =
649 llvm::StructType::get(Context&: CGM.getLLVMContext(), Elements: elementTypes, isPacked: true);
650 info.CanBeGlobal = true;
651 info.buildCaptureMap();
652 return;
653 }
654
655 // Sort the layout by alignment. We have to use a stable sort here
656 // to get reproducible results. There should probably be an
657 // llvm::array_pod_stable_sort.
658 llvm::stable_sort(Range&: layout);
659
660 // Needed for blocks layout info.
661 info.BlockHeaderForcedGapOffset = info.BlockSize;
662 info.BlockHeaderForcedGapSize = CharUnits::Zero();
663
664 CharUnits &blockSize = info.BlockSize;
665 info.BlockAlign = std::max(a: maxFieldAlign, b: info.BlockAlign);
666
667 // Assuming that the first byte in the header is maximally aligned,
668 // get the alignment of the first byte following the header.
669 CharUnits endAlign = getLowBit(v: blockSize);
670
671 // If the end of the header isn't satisfactorily aligned for the
672 // maximum thing, look for things that are okay with the header-end
673 // alignment, and keep appending them until we get something that's
674 // aligned right. This algorithm is only guaranteed optimal if
675 // that condition is satisfied at some point; otherwise we can get
676 // things like:
677 // header // next byte has alignment 4
678 // something_with_size_5; // next byte has alignment 1
679 // something_with_alignment_8;
680 // which has 7 bytes of padding, as opposed to the naive solution
681 // which might have less (?).
682 if (endAlign < maxFieldAlign) {
683 SmallVectorImpl<BlockLayoutChunk>::iterator
684 li = layout.begin() + 1, le = layout.end();
685
686 // Look for something that the header end is already
687 // satisfactorily aligned for.
688 for (; li != le && endAlign < li->Alignment; ++li)
689 ;
690
691 // If we found something that's naturally aligned for the end of
692 // the header, keep adding things...
693 if (li != le) {
694 SmallVectorImpl<BlockLayoutChunk>::iterator first = li;
695 for (; li != le; ++li) {
696 assert(endAlign >= li->Alignment);
697
698 li->setIndex(info, index: elementTypes.size(), offset: blockSize);
699 elementTypes.push_back(Elt: li->Type);
700 blockSize += li->Size;
701 endAlign = getLowBit(v: blockSize);
702
703 // ...until we get to the alignment of the maximum field.
704 if (endAlign >= maxFieldAlign) {
705 ++li;
706 break;
707 }
708 }
709 // Don't re-append everything we just appended.
710 layout.erase(CS: first, CE: li);
711 }
712 }
713
714 assert(endAlign == getLowBit(blockSize));
715
716 // At this point, we just have to add padding if the end align still
717 // isn't aligned right.
718 if (endAlign < maxFieldAlign) {
719 CharUnits newBlockSize = blockSize.alignTo(Align: maxFieldAlign);
720 CharUnits padding = newBlockSize - blockSize;
721
722 // If we haven't yet added any fields, remember that there was an
723 // initial gap; this need to go into the block layout bit map.
724 if (blockSize == info.BlockHeaderForcedGapOffset) {
725 info.BlockHeaderForcedGapSize = padding;
726 }
727
728 elementTypes.push_back(Elt: llvm::ArrayType::get(ElementType: CGM.Int8Ty,
729 NumElements: padding.getQuantity()));
730 blockSize = newBlockSize;
731 endAlign = getLowBit(v: blockSize); // might be > maxFieldAlign
732 }
733
734 assert(endAlign >= maxFieldAlign);
735 assert(endAlign == getLowBit(blockSize));
736 // Slam everything else on now. This works because they have
737 // strictly decreasing alignment and we expect that size is always a
738 // multiple of alignment.
739 for (SmallVectorImpl<BlockLayoutChunk>::iterator
740 li = layout.begin(), le = layout.end(); li != le; ++li) {
741 if (endAlign < li->Alignment) {
742 // size may not be multiple of alignment. This can only happen with
743 // an over-aligned variable. We will be adding a padding field to
744 // make the size be multiple of alignment.
745 CharUnits padding = li->Alignment - endAlign;
746 elementTypes.push_back(Elt: llvm::ArrayType::get(ElementType: CGM.Int8Ty,
747 NumElements: padding.getQuantity()));
748 blockSize += padding;
749 endAlign = getLowBit(v: blockSize);
750 }
751 assert(endAlign >= li->Alignment);
752 li->setIndex(info, index: elementTypes.size(), offset: blockSize);
753 elementTypes.push_back(Elt: li->Type);
754 blockSize += li->Size;
755 endAlign = getLowBit(v: blockSize);
756 }
757
758 info.buildCaptureMap();
759 info.StructureType =
760 llvm::StructType::get(Context&: CGM.getLLVMContext(), Elements: elementTypes, isPacked: true);
761}
762
763/// Emit a block literal expression in the current function.
764llvm::Value *CodeGenFunction::EmitBlockLiteral(const BlockExpr *blockExpr) {
765 // If the block has no captures, we won't have a pre-computed
766 // layout for it.
767 if (!blockExpr->getBlockDecl()->hasCaptures())
768 // The block literal is emitted as a global variable, and the block invoke
769 // function has to be extracted from its initializer.
770 if (llvm::Constant *Block = CGM.getAddrOfGlobalBlockIfEmitted(BE: blockExpr))
771 return Block;
772
773 CGBlockInfo blockInfo(blockExpr->getBlockDecl(), CurFn->getName());
774 computeBlockInfo(CGM, CGF: this, info&: blockInfo);
775 blockInfo.BlockExpression = blockExpr;
776 if (!blockInfo.CanBeGlobal)
777 blockInfo.LocalAddress = CreateTempAlloca(Ty: blockInfo.StructureType,
778 align: blockInfo.BlockAlign, Name: "block");
779 return EmitBlockLiteral(Info: blockInfo);
780}
781
782llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) {
783 bool IsOpenCL = CGM.getContext().getLangOpts().OpenCL;
784 llvm::PointerType *GenVoidPtrTy =
785 IsOpenCL ? CGM.getOpenCLRuntime().getGenericVoidPointerType() : VoidPtrTy;
786 LangAS GenVoidPtrAddr = IsOpenCL ? LangAS::opencl_generic : LangAS::Default;
787 auto GenVoidPtrSize = CharUnits::fromQuantity(
788 Quantity: CGM.getTarget().getPointerWidth(AddrSpace: GenVoidPtrAddr) / 8);
789 // Using the computed layout, generate the actual block function.
790 bool isLambdaConv = blockInfo.getBlockDecl()->isConversionFromLambda();
791 CodeGenFunction BlockCGF{CGM, true};
792 BlockCGF.SanOpts = SanOpts;
793 auto *InvokeFn = BlockCGF.GenerateBlockFunction(
794 GD: CurGD, Info: blockInfo, ldm: LocalDeclMap, IsLambdaConversionToBlock: isLambdaConv, BuildGlobalBlock: blockInfo.CanBeGlobal);
795 auto *blockFn = llvm::ConstantExpr::getPointerCast(C: InvokeFn, Ty: GenVoidPtrTy);
796
797 // If there is nothing to capture, we can emit this as a global block.
798 if (blockInfo.CanBeGlobal)
799 return CGM.getAddrOfGlobalBlockIfEmitted(BE: blockInfo.BlockExpression);
800
801 // Otherwise, we have to emit this as a local block.
802
803 RawAddress blockAddr = blockInfo.LocalAddress;
804 assert(blockAddr.isValid() && "block has no address!");
805
806 llvm::Constant *isa;
807 llvm::Constant *descriptor;
808 BlockFlags flags;
809 if (!IsOpenCL) {
810 // If the block is non-escaping, set field 'isa 'to NSConcreteGlobalBlock
811 // and set the BLOCK_IS_GLOBAL bit of field 'flags'. Copying a non-escaping
812 // block just returns the original block and releasing it is a no-op.
813 llvm::Constant *blockISA = blockInfo.NoEscape
814 ? CGM.getNSConcreteGlobalBlock()
815 : CGM.getNSConcreteStackBlock();
816 isa = blockISA;
817
818 // Compute the initial on-stack block flags.
819 if (!CGM.getCodeGenOpts().DisableBlockSignatureString)
820 flags = BLOCK_HAS_SIGNATURE;
821 if (blockInfo.HasCapturedVariableLayout)
822 flags |= BLOCK_HAS_EXTENDED_LAYOUT;
823 if (blockInfo.NeedsCopyDispose)
824 flags |= BLOCK_HAS_COPY_DISPOSE;
825 if (blockInfo.HasCXXObject)
826 flags |= BLOCK_HAS_CXX_OBJ;
827 if (blockInfo.UsesStret)
828 flags |= BLOCK_USE_STRET;
829 if (blockInfo.NoEscape)
830 flags |= BLOCK_IS_NOESCAPE | BLOCK_IS_GLOBAL;
831
832 // Build the block descriptor.
833 descriptor = buildBlockDescriptor(CGM, blockInfo);
834 }
835
836 auto projectField = [&](unsigned index, const Twine &name) -> Address {
837 return Builder.CreateStructGEP(Addr: blockAddr, Index: index, Name: name);
838 };
839 auto storeField = [&](llvm::Value *value, unsigned index, const Twine &name) {
840 Builder.CreateStore(Val: value, Addr: projectField(index, name));
841 };
842
843 // Initialize the block header.
844 {
845 // We assume all the header fields are densely packed.
846 unsigned index = 0;
847 CharUnits offset;
848 auto addHeaderField = [&](llvm::Value *value, CharUnits size,
849 const Twine &name) {
850 storeField(value, index, name);
851 offset += size;
852 index++;
853 };
854 auto addSignedHeaderField =
855 [&](llvm::Value *Value, const PointerAuthSchema &Schema,
856 GlobalDecl Decl, QualType Type, CharUnits Size, const Twine &Name) {
857 auto StorageAddress = projectField(index, Name);
858 if (Schema) {
859 auto AuthInfo = EmitPointerAuthInfo(
860 Schema, StorageAddress: StorageAddress.emitRawPointer(CGF&: *this), SchemaDecl: Decl, SchemaType: Type);
861 Value = EmitPointerAuthSign(Info: AuthInfo, Pointer: Value);
862 }
863 Builder.CreateStore(Val: Value, Addr: StorageAddress);
864 offset += Size;
865 index++;
866 };
867
868 if (!IsOpenCL) {
869 addSignedHeaderField(
870 isa, CGM.getCodeGenOpts().PointerAuth.ObjCIsaPointers, GlobalDecl(),
871 QualType(), getPointerSize(), "block.isa");
872 addHeaderField(llvm::ConstantInt::get(Ty: IntTy, V: flags.getBitMask()),
873 getIntSize(), "block.flags");
874 addHeaderField(llvm::ConstantInt::get(Ty: IntTy, V: 0), getIntSize(),
875 "block.reserved");
876 } else {
877 addHeaderField(
878 llvm::ConstantInt::get(Ty: IntTy, V: blockInfo.BlockSize.getQuantity()),
879 getIntSize(), "block.size");
880 addHeaderField(
881 llvm::ConstantInt::get(Ty: IntTy, V: blockInfo.BlockAlign.getQuantity()),
882 getIntSize(), "block.align");
883 }
884
885 if (!IsOpenCL) {
886 llvm::Value *blockFnPtr =
887 llvm::ConstantExpr::getBitCast(C: InvokeFn, Ty: VoidPtrTy);
888 QualType type = blockInfo.getBlockExpr()
889 ->getType()
890 ->castAs<BlockPointerType>()
891 ->getPointeeType();
892 addSignedHeaderField(
893 blockFnPtr,
894 CGM.getCodeGenOpts().PointerAuth.BlockInvocationFunctionPointers,
895 GlobalDecl(), type, getPointerSize(), "block.invoke");
896
897 addSignedHeaderField(
898 descriptor, CGM.getCodeGenOpts().PointerAuth.BlockDescriptorPointers,
899 GlobalDecl(), type, getPointerSize(), "block.descriptor");
900 } else if (auto *Helper =
901 CGM.getTargetCodeGenInfo().getTargetOpenCLBlockHelper()) {
902 addHeaderField(blockFn, GenVoidPtrSize, "block.invoke");
903 for (auto I : Helper->getCustomFieldValues(CGF&: *this, Info: blockInfo)) {
904 addHeaderField(
905 I.first,
906 CharUnits::fromQuantity(
907 Quantity: CGM.getDataLayout().getTypeAllocSize(Ty: I.first->getType())),
908 I.second);
909 }
910 } else
911 addHeaderField(blockFn, GenVoidPtrSize, "block.invoke");
912 }
913
914 // Finally, capture all the values into the block.
915 const BlockDecl *blockDecl = blockInfo.getBlockDecl();
916
917 // First, 'this'.
918 if (blockDecl->capturesCXXThis()) {
919 Address addr =
920 projectField(blockInfo.CXXThisIndex, "block.captured-this.addr");
921 Builder.CreateStore(Val: LoadCXXThis(), Addr: addr);
922 }
923
924 // Next, captured variables.
925 for (const auto &CI : blockDecl->captures()) {
926 const VarDecl *variable = CI.getVariable();
927 const CGBlockInfo::Capture &capture = blockInfo.getCapture(var: variable);
928
929 // Ignore constant captures.
930 if (capture.isConstant()) continue;
931
932 QualType type = capture.fieldType();
933
934 // This will be a [[type]]*, except that a byref entry will just be
935 // an i8**.
936 Address blockField = projectField(capture.getIndex(), "block.captured");
937
938 // Compute the address of the thing we're going to move into the
939 // block literal.
940 Address src = Address::invalid();
941
942 if (blockDecl->isConversionFromLambda()) {
943 // The lambda capture in a lambda's conversion-to-block-pointer is
944 // special; we'll simply emit it directly.
945 src = Address::invalid();
946 } else if (CI.isEscapingByref()) {
947 if (BlockInfo && CI.isNested()) {
948 // We need to use the capture from the enclosing block.
949 const CGBlockInfo::Capture &enclosingCapture =
950 BlockInfo->getCapture(var: variable);
951
952 // This is a [[type]]*, except that a byref entry will just be an i8**.
953 src = Builder.CreateStructGEP(Addr: LoadBlockStruct(),
954 Index: enclosingCapture.getIndex(),
955 Name: "block.capture.addr");
956 } else {
957 auto I = LocalDeclMap.find(Val: variable);
958 assert(I != LocalDeclMap.end());
959 src = I->second;
960 }
961 } else {
962 DeclRefExpr declRef(getContext(), const_cast<VarDecl *>(variable),
963 /*RefersToEnclosingVariableOrCapture*/ CI.isNested(),
964 type.getNonReferenceType(), VK_LValue,
965 SourceLocation());
966 src = EmitDeclRefLValue(E: &declRef).getAddress();
967 };
968
969 // For byrefs, we just write the pointer to the byref struct into
970 // the block field. There's no need to chase the forwarding
971 // pointer at this point, since we're building something that will
972 // live a shorter life than the stack byref anyway.
973 if (CI.isEscapingByref()) {
974 // Get a void* that points to the byref struct.
975 llvm::Value *byrefPointer;
976 if (CI.isNested())
977 byrefPointer = Builder.CreateLoad(Addr: src, Name: "byref.capture");
978 else
979 byrefPointer = src.emitRawPointer(CGF&: *this);
980
981 // Write that void* into the capture field.
982 Builder.CreateStore(Val: byrefPointer, Addr: blockField);
983
984 // If we have a copy constructor, evaluate that into the block field.
985 } else if (const Expr *copyExpr = CI.getCopyExpr()) {
986 if (blockDecl->isConversionFromLambda()) {
987 // If we have a lambda conversion, emit the expression
988 // directly into the block instead.
989 AggValueSlot Slot =
990 AggValueSlot::forAddr(addr: blockField, quals: Qualifiers(),
991 isDestructed: AggValueSlot::IsDestructed,
992 needsGC: AggValueSlot::DoesNotNeedGCBarriers,
993 isAliased: AggValueSlot::IsNotAliased,
994 mayOverlap: AggValueSlot::DoesNotOverlap);
995 EmitAggExpr(E: copyExpr, AS: Slot);
996 } else {
997 EmitSynthesizedCXXCopyCtor(Dest: blockField, Src: src, Exp: copyExpr);
998 }
999
1000 // If it's a reference variable, copy the reference into the block field.
1001 } else if (type->getAs<ReferenceType>()) {
1002 Builder.CreateStore(Val: src.emitRawPointer(CGF&: *this), Addr: blockField);
1003
1004 // If type is const-qualified, copy the value into the block field.
1005 } else if (type.isConstQualified() &&
1006 type.getObjCLifetime() == Qualifiers::OCL_Strong &&
1007 CGM.getCodeGenOpts().OptimizationLevel != 0) {
1008 llvm::Value *value = Builder.CreateLoad(Addr: src, Name: "captured");
1009 Builder.CreateStore(Val: value, Addr: blockField);
1010
1011 // If this is an ARC __strong block-pointer variable, don't do a
1012 // block copy.
1013 //
1014 // TODO: this can be generalized into the normal initialization logic:
1015 // we should never need to do a block-copy when initializing a local
1016 // variable, because the local variable's lifetime should be strictly
1017 // contained within the stack block's.
1018 } else if (type.getObjCLifetime() == Qualifiers::OCL_Strong &&
1019 type->isBlockPointerType()) {
1020 // Load the block and do a simple retain.
1021 llvm::Value *value = Builder.CreateLoad(Addr: src, Name: "block.captured_block");
1022 value = EmitARCRetainNonBlock(value);
1023
1024 // Do a primitive store to the block field.
1025 Builder.CreateStore(Val: value, Addr: blockField);
1026
1027 // Otherwise, fake up a POD copy into the block field.
1028 } else {
1029 // Fake up a new variable so that EmitScalarInit doesn't think
1030 // we're referring to the variable in its own initializer.
1031 ImplicitParamDecl BlockFieldPseudoVar(getContext(), type,
1032 ImplicitParamKind::Other);
1033
1034 // We use one of these or the other depending on whether the
1035 // reference is nested.
1036 DeclRefExpr declRef(getContext(), const_cast<VarDecl *>(variable),
1037 /*RefersToEnclosingVariableOrCapture*/ CI.isNested(),
1038 type, VK_LValue, SourceLocation());
1039
1040 ImplicitCastExpr l2r(ImplicitCastExpr::OnStack, type, CK_LValueToRValue,
1041 &declRef, VK_PRValue, FPOptionsOverride());
1042 // FIXME: Pass a specific location for the expr init so that the store is
1043 // attributed to a reasonable location - otherwise it may be attributed to
1044 // locations of subexpressions in the initialization.
1045 EmitExprAsInit(init: &l2r, D: &BlockFieldPseudoVar,
1046 lvalue: MakeAddrLValue(Addr: blockField, T: type, Source: AlignmentSource::Decl),
1047 /*captured by init*/ capturedByInit: false);
1048 }
1049
1050 // Push a cleanup for the capture if necessary.
1051 if (!blockInfo.NoEscape && !blockInfo.NeedsCopyDispose)
1052 continue;
1053
1054 // Ignore __block captures; there's nothing special in the on-stack block
1055 // that we need to do for them.
1056 if (CI.isByRef())
1057 continue;
1058
1059 // Ignore objects that aren't destructed.
1060 QualType::DestructionKind dtorKind = type.isDestructedType();
1061 if (dtorKind == QualType::DK_none)
1062 continue;
1063
1064 CodeGenFunction::Destroyer *destroyer;
1065
1066 // Block captures count as local values and have imprecise semantics.
1067 // They also can't be arrays, so need to worry about that.
1068 //
1069 // For const-qualified captures, emit clang.arc.use to ensure the captured
1070 // object doesn't get released while we are still depending on its validity
1071 // within the block.
1072 if (type.isConstQualified() &&
1073 type.getObjCLifetime() == Qualifiers::OCL_Strong &&
1074 CGM.getCodeGenOpts().OptimizationLevel != 0) {
1075 assert(CGM.getLangOpts().ObjCAutoRefCount &&
1076 "expected ObjC ARC to be enabled");
1077 destroyer = emitARCIntrinsicUse;
1078 } else if (dtorKind == QualType::DK_objc_strong_lifetime) {
1079 destroyer = destroyARCStrongImprecise;
1080 } else {
1081 destroyer = getDestroyer(destructionKind: dtorKind);
1082 }
1083
1084 CleanupKind cleanupKind = NormalCleanup;
1085 bool useArrayEHCleanup = needsEHCleanup(kind: dtorKind);
1086 if (useArrayEHCleanup)
1087 cleanupKind = NormalAndEHCleanup;
1088
1089 // Extend the lifetime of the capture to the end of the scope enclosing the
1090 // block expression except when the block decl is in the list of RetExpr's
1091 // cleanup objects, in which case its lifetime ends after the full
1092 // expression.
1093 auto IsBlockDeclInRetExpr = [&]() {
1094 auto *EWC = llvm::dyn_cast_or_null<ExprWithCleanups>(Val: RetExpr);
1095 if (EWC)
1096 for (auto &C : EWC->getObjects())
1097 if (auto *BD = C.dyn_cast<BlockDecl *>())
1098 if (BD == blockDecl)
1099 return true;
1100 return false;
1101 };
1102
1103 if (IsBlockDeclInRetExpr())
1104 pushDestroy(kind: cleanupKind, addr: blockField, type, destroyer, useEHCleanupForArray: useArrayEHCleanup);
1105 else
1106 pushLifetimeExtendedDestroy(kind: cleanupKind, addr: blockField, type, destroyer,
1107 useEHCleanupForArray: useArrayEHCleanup);
1108 }
1109
1110 // Cast to the converted block-pointer type, which happens (somewhat
1111 // unfortunately) to be a pointer to function type.
1112 llvm::Value *result = Builder.CreatePointerCast(
1113 V: blockAddr.getPointer(), DestTy: ConvertType(T: blockInfo.getBlockExpr()->getType()));
1114
1115 if (IsOpenCL) {
1116 CGM.getOpenCLRuntime().recordBlockInfo(E: blockInfo.BlockExpression, InvokeF: InvokeFn,
1117 Block: result, BlockTy: blockInfo.StructureType);
1118 }
1119
1120 return result;
1121}
1122
1123
1124llvm::Type *CodeGenModule::getBlockDescriptorType() {
1125 if (BlockDescriptorType)
1126 return BlockDescriptorType;
1127
1128 unsigned AddrSpace = 0;
1129 if (getLangOpts().OpenCL)
1130 AddrSpace = getContext().getTargetAddressSpace(AS: LangAS::opencl_constant);
1131 BlockDescriptorType = llvm::PointerType::get(C&: getLLVMContext(), AddressSpace: AddrSpace);
1132 return BlockDescriptorType;
1133}
1134
1135llvm::Type *CodeGenModule::getGenericBlockLiteralType() {
1136 if (GenericBlockLiteralType)
1137 return GenericBlockLiteralType;
1138
1139 llvm::Type *BlockDescPtrTy = getBlockDescriptorType();
1140
1141 if (getLangOpts().OpenCL) {
1142 // struct __opencl_block_literal_generic {
1143 // int __size;
1144 // int __align;
1145 // __generic void *__invoke;
1146 // /* custom fields */
1147 // };
1148 SmallVector<llvm::Type *, 8> StructFields(
1149 {IntTy, IntTy, getOpenCLRuntime().getGenericVoidPointerType()});
1150 if (auto *Helper = getTargetCodeGenInfo().getTargetOpenCLBlockHelper()) {
1151 llvm::append_range(C&: StructFields, R: Helper->getCustomFieldTypes());
1152 }
1153 GenericBlockLiteralType = llvm::StructType::create(
1154 Elements: StructFields, Name: "struct.__opencl_block_literal_generic");
1155 } else {
1156 // struct __block_literal_generic {
1157 // void *__isa;
1158 // int __flags;
1159 // int __reserved;
1160 // void (*__invoke)(void *);
1161 // struct __block_descriptor *__descriptor;
1162 // };
1163 GenericBlockLiteralType =
1164 llvm::StructType::create(Name: "struct.__block_literal_generic", elt1: VoidPtrTy,
1165 elts: IntTy, elts: IntTy, elts: VoidPtrTy, elts: BlockDescPtrTy);
1166 }
1167
1168 return GenericBlockLiteralType;
1169}
1170
1171RValue CodeGenFunction::EmitBlockCallExpr(const CallExpr *E,
1172 ReturnValueSlot ReturnValue,
1173 llvm::CallBase **CallOrInvoke) {
1174 const auto *BPT = E->getCallee()->getType()->castAs<BlockPointerType>();
1175 llvm::Value *BlockPtr = EmitScalarExpr(E: E->getCallee());
1176 llvm::Type *GenBlockTy = CGM.getGenericBlockLiteralType();
1177 llvm::Value *Func = nullptr;
1178 QualType FnType = BPT->getPointeeType();
1179 ASTContext &Ctx = getContext();
1180 CallArgList Args;
1181
1182 llvm::Value *FuncPtr = nullptr;
1183
1184 if (getLangOpts().OpenCL) {
1185 // For OpenCL, BlockPtr is already casted to generic block literal.
1186
1187 // First argument of a block call is a generic block literal casted to
1188 // generic void pointer, i.e. i8 addrspace(4)*
1189 llvm::Type *GenericVoidPtrTy =
1190 CGM.getOpenCLRuntime().getGenericVoidPointerType();
1191 llvm::Value *BlockDescriptor = Builder.CreatePointerCast(
1192 V: BlockPtr, DestTy: GenericVoidPtrTy);
1193 QualType VoidPtrQualTy = Ctx.getPointerType(
1194 T: Ctx.getAddrSpaceQualType(T: Ctx.VoidTy, AddressSpace: LangAS::opencl_generic));
1195 Args.add(rvalue: RValue::get(V: BlockDescriptor), type: VoidPtrQualTy);
1196 // And the rest of the arguments.
1197 EmitCallArgs(Args, Prototype: FnType->getAs<FunctionProtoType>(), ArgRange: E->arguments());
1198
1199 // We *can* call the block directly unless it is a function argument.
1200 if (!isa<ParmVarDecl>(Val: E->getCalleeDecl()))
1201 Func = CGM.getOpenCLRuntime().getInvokeFunction(E: E->getCallee());
1202 else {
1203 FuncPtr = Builder.CreateStructGEP(Ty: GenBlockTy, Ptr: BlockPtr, Idx: 2);
1204 Func = Builder.CreateAlignedLoad(Ty: GenericVoidPtrTy, Addr: FuncPtr,
1205 Align: getPointerAlign());
1206 }
1207 } else {
1208 // Bitcast the block literal to a generic block literal.
1209 BlockPtr =
1210 Builder.CreatePointerCast(V: BlockPtr, DestTy: DefaultPtrTy, Name: "block.literal");
1211 // Get pointer to the block invoke function
1212 FuncPtr = Builder.CreateStructGEP(Ty: GenBlockTy, Ptr: BlockPtr, Idx: 3);
1213
1214 // First argument is a block literal casted to a void pointer
1215 BlockPtr = Builder.CreatePointerCast(V: BlockPtr, DestTy: VoidPtrTy);
1216 Args.add(rvalue: RValue::get(V: BlockPtr), type: Ctx.VoidPtrTy);
1217 // And the rest of the arguments.
1218 EmitCallArgs(Args, Prototype: FnType->getAs<FunctionProtoType>(), ArgRange: E->arguments());
1219
1220 // Load the function.
1221 Func = Builder.CreateAlignedLoad(Ty: VoidPtrTy, Addr: FuncPtr, Align: getPointerAlign());
1222 }
1223
1224 const FunctionType *FuncTy = FnType->castAs<FunctionType>();
1225 const CGFunctionInfo &FnInfo =
1226 CGM.getTypes().arrangeBlockFunctionCall(args: Args, type: FuncTy);
1227
1228 // Prepare the callee.
1229 CGPointerAuthInfo PointerAuth;
1230 if (auto &AuthSchema =
1231 CGM.getCodeGenOpts().PointerAuth.BlockInvocationFunctionPointers) {
1232 assert(FuncPtr != nullptr && "Missing function pointer for AuthInfo");
1233 PointerAuth =
1234 EmitPointerAuthInfo(Schema: AuthSchema, StorageAddress: FuncPtr, SchemaDecl: GlobalDecl(), SchemaType: FnType);
1235 }
1236
1237 CGCallee Callee(CGCalleeInfo(), Func, PointerAuth);
1238
1239 // And call the block.
1240 return EmitCall(CallInfo: FnInfo, Callee, ReturnValue, Args, CallOrInvoke);
1241}
1242
1243Address CodeGenFunction::GetAddrOfBlockDecl(const VarDecl *variable) {
1244 assert(BlockInfo && "evaluating block ref without block information?");
1245 const CGBlockInfo::Capture &capture = BlockInfo->getCapture(var: variable);
1246
1247 // Handle constant captures.
1248 if (capture.isConstant()) return LocalDeclMap.find(Val: variable)->second;
1249
1250 Address addr = Builder.CreateStructGEP(Addr: LoadBlockStruct(), Index: capture.getIndex(),
1251 Name: "block.capture.addr");
1252
1253 if (variable->isEscapingByref()) {
1254 // addr should be a void** right now. Load, then cast the result
1255 // to byref*.
1256
1257 auto &byrefInfo = getBlockByrefInfo(var: variable);
1258 addr = Address(Builder.CreateLoad(Addr: addr), byrefInfo.Type,
1259 byrefInfo.ByrefAlignment);
1260
1261 addr = emitBlockByrefAddress(baseAddr: addr, info: byrefInfo, /*follow*/ followForward: true,
1262 name: variable->getName());
1263 }
1264
1265 assert((!variable->isNonEscapingByref() ||
1266 capture.fieldType()->isReferenceType()) &&
1267 "the capture field of a non-escaping variable should have a "
1268 "reference type");
1269 if (capture.fieldType()->isReferenceType())
1270 addr = EmitLoadOfReference(RefLVal: MakeAddrLValue(Addr: addr, T: capture.fieldType()));
1271
1272 return addr;
1273}
1274
1275void CodeGenModule::setAddrOfGlobalBlock(const BlockExpr *BE,
1276 llvm::Constant *Addr) {
1277 bool Ok = EmittedGlobalBlocks.insert(KV: std::make_pair(x&: BE, y&: Addr)).second;
1278 (void)Ok;
1279 assert(Ok && "Trying to replace an already-existing global block!");
1280}
1281
1282llvm::Constant *
1283CodeGenModule::GetAddrOfGlobalBlock(const BlockExpr *BE,
1284 StringRef Name) {
1285 if (llvm::Constant *Block = getAddrOfGlobalBlockIfEmitted(BE))
1286 return Block;
1287
1288 CGBlockInfo blockInfo(BE->getBlockDecl(), Name);
1289 blockInfo.BlockExpression = BE;
1290
1291 // Compute information about the layout, etc., of this block.
1292 computeBlockInfo(CGM&: *this, CGF: nullptr, info&: blockInfo);
1293
1294 // Using that metadata, generate the actual block function.
1295 {
1296 CodeGenFunction::DeclMapTy LocalDeclMap;
1297 CodeGenFunction(*this).GenerateBlockFunction(
1298 GD: GlobalDecl(), Info: blockInfo, ldm: LocalDeclMap,
1299 /*IsLambdaConversionToBlock*/ false, /*BuildGlobalBlock*/ true);
1300 }
1301
1302 return getAddrOfGlobalBlockIfEmitted(BE);
1303}
1304
1305static llvm::Constant *buildGlobalBlock(CodeGenModule &CGM,
1306 const CGBlockInfo &blockInfo,
1307 llvm::Constant *blockFn) {
1308 assert(blockInfo.CanBeGlobal);
1309 // Callers should detect this case on their own: calling this function
1310 // generally requires computing layout information, which is a waste of time
1311 // if we've already emitted this block.
1312 assert(!CGM.getAddrOfGlobalBlockIfEmitted(blockInfo.BlockExpression) &&
1313 "Refusing to re-emit a global block.");
1314
1315 // Generate the constants for the block literal initializer.
1316 ConstantInitBuilder builder(CGM);
1317 auto fields = builder.beginStruct();
1318
1319 bool IsOpenCL = CGM.getLangOpts().OpenCL;
1320 bool IsWindows = CGM.getTarget().getTriple().isOSWindows();
1321 auto &CGOPointerAuth = CGM.getCodeGenOpts().PointerAuth;
1322 if (!IsOpenCL) {
1323 // isa
1324 if (IsWindows)
1325 fields.addNullPointer(ptrTy: CGM.Int8PtrPtrTy);
1326 else
1327 fields.addSignedPointer(Pointer: CGM.getNSConcreteGlobalBlock(),
1328 Schema: CGOPointerAuth.ObjCIsaPointers, CalleeDecl: GlobalDecl(),
1329 CalleeType: QualType());
1330
1331 // __flags
1332 BlockFlags flags = BLOCK_IS_GLOBAL;
1333 if (!CGM.getCodeGenOpts().DisableBlockSignatureString)
1334 flags |= BLOCK_HAS_SIGNATURE;
1335 if (blockInfo.UsesStret)
1336 flags |= BLOCK_USE_STRET;
1337
1338 fields.addInt(intTy: CGM.IntTy, value: flags.getBitMask());
1339
1340 // Reserved
1341 fields.addInt(intTy: CGM.IntTy, value: 0);
1342 } else {
1343 fields.addInt(intTy: CGM.IntTy, value: blockInfo.BlockSize.getQuantity());
1344 fields.addInt(intTy: CGM.IntTy, value: blockInfo.BlockAlign.getQuantity());
1345 }
1346
1347 // Function
1348 if (auto &Schema = CGOPointerAuth.BlockInvocationFunctionPointers) {
1349 QualType FnType = blockInfo.getBlockExpr()
1350 ->getType()
1351 ->castAs<BlockPointerType>()
1352 ->getPointeeType();
1353 fields.addSignedPointer(Pointer: blockFn, Schema, CalleeDecl: GlobalDecl(), CalleeType: FnType);
1354 } else
1355 fields.add(value: blockFn);
1356
1357 if (!IsOpenCL) {
1358 // Descriptor
1359 llvm::Constant *Descriptor = buildBlockDescriptor(CGM, blockInfo);
1360 fields.addSignedPointer(Pointer: Descriptor, Schema: CGOPointerAuth.BlockDescriptorPointers,
1361 CalleeDecl: GlobalDecl(), CalleeType: QualType());
1362 } else if (auto *Helper =
1363 CGM.getTargetCodeGenInfo().getTargetOpenCLBlockHelper()) {
1364 for (auto *I : Helper->getCustomFieldValues(CGM, Info: blockInfo)) {
1365 fields.add(value: I);
1366 }
1367 }
1368
1369 unsigned AddrSpace = 0;
1370 if (CGM.getContext().getLangOpts().OpenCL)
1371 AddrSpace = CGM.getContext().getTargetAddressSpace(AS: LangAS::opencl_global);
1372
1373 llvm::GlobalVariable *literal = fields.finishAndCreateGlobal(
1374 args: "__block_literal_global", args: blockInfo.BlockAlign,
1375 /*constant*/ args: !IsWindows, args: llvm::GlobalVariable::InternalLinkage, args&: AddrSpace);
1376
1377 literal->addAttribute(Kind: "objc_arc_inert");
1378
1379 // Windows does not allow globals to be initialised to point to globals in
1380 // different DLLs. Any such variables must run code to initialise them.
1381 if (IsWindows) {
1382 auto *Init = llvm::Function::Create(Ty: llvm::FunctionType::get(Result: CGM.VoidTy,
1383 isVarArg: {}), Linkage: llvm::GlobalValue::InternalLinkage, N: ".block_isa_init",
1384 M: &CGM.getModule());
1385 llvm::IRBuilder<> b(llvm::BasicBlock::Create(Context&: CGM.getLLVMContext(), Name: "entry",
1386 Parent: Init));
1387 b.CreateAlignedStore(Val: CGM.getNSConcreteGlobalBlock(),
1388 Ptr: b.CreateStructGEP(Ty: literal->getValueType(), Ptr: literal, Idx: 0),
1389 Align: CGM.getPointerAlign().getAsAlign());
1390 b.CreateRetVoid();
1391 // We can't use the normal LLVM global initialisation array, because we
1392 // need to specify that this runs early in library initialisation.
1393 auto *InitVar = new llvm::GlobalVariable(CGM.getModule(), Init->getType(),
1394 /*isConstant*/true, llvm::GlobalValue::InternalLinkage,
1395 Init, ".block_isa_init_ptr");
1396 InitVar->setSection(".CRT$XCLa");
1397 CGM.addUsedGlobal(GV: InitVar);
1398 }
1399
1400 // Return a constant of the appropriately-casted type.
1401 llvm::Type *RequiredType =
1402 CGM.getTypes().ConvertType(T: blockInfo.getBlockExpr()->getType());
1403 llvm::Constant *Result =
1404 llvm::ConstantExpr::getPointerCast(C: literal, Ty: RequiredType);
1405 CGM.setAddrOfGlobalBlock(BE: blockInfo.BlockExpression, Addr: Result);
1406 if (CGM.getContext().getLangOpts().OpenCL)
1407 CGM.getOpenCLRuntime().recordBlockInfo(
1408 E: blockInfo.BlockExpression,
1409 InvokeF: cast<llvm::Function>(Val: blockFn->stripPointerCasts()), Block: Result,
1410 BlockTy: literal->getValueType());
1411 return Result;
1412}
1413
1414void CodeGenFunction::setBlockContextParameter(const ImplicitParamDecl *D,
1415 unsigned argNum,
1416 llvm::Value *arg) {
1417 assert(BlockInfo && "not emitting prologue of block invocation function?!");
1418
1419 // Allocate a stack slot like for any local variable to guarantee optimal
1420 // debug info at -O0. The mem2reg pass will eliminate it when optimizing.
1421 RawAddress alloc = CreateMemTemp(T: D->getType(), Name: D->getName() + ".addr");
1422 Builder.CreateStore(Val: arg, Addr: alloc);
1423 if (CGDebugInfo *DI = getDebugInfo()) {
1424 if (CGM.getCodeGenOpts().hasReducedDebugInfo()) {
1425 DI->setLocation(D->getLocation());
1426 DI->EmitDeclareOfBlockLiteralArgVariable(
1427 block: *BlockInfo, Name: D->getName(), ArgNo: argNum,
1428 LocalAddr: cast<llvm::AllocaInst>(Val: alloc.getPointer()->stripPointerCasts()),
1429 Builder);
1430 }
1431 }
1432
1433 SourceLocation StartLoc = BlockInfo->getBlockExpr()->getBody()->getBeginLoc();
1434 ApplyDebugLocation Scope(*this, StartLoc);
1435
1436 // Instead of messing around with LocalDeclMap, just set the value
1437 // directly as BlockPointer.
1438 BlockPointer = Builder.CreatePointerCast(
1439 V: arg,
1440 DestTy: llvm::PointerType::get(
1441 C&: getLLVMContext(),
1442 AddressSpace: getContext().getLangOpts().OpenCL
1443 ? getContext().getTargetAddressSpace(AS: LangAS::opencl_generic)
1444 : 0),
1445 Name: "block");
1446}
1447
1448Address CodeGenFunction::LoadBlockStruct() {
1449 assert(BlockInfo && "not in a block invocation function!");
1450 assert(BlockPointer && "no block pointer set!");
1451 return Address(BlockPointer, BlockInfo->StructureType, BlockInfo->BlockAlign);
1452}
1453
1454llvm::Function *CodeGenFunction::GenerateBlockFunction(
1455 GlobalDecl GD, const CGBlockInfo &blockInfo, const DeclMapTy &ldm,
1456 bool IsLambdaConversionToBlock, bool BuildGlobalBlock) {
1457 const BlockDecl *blockDecl = blockInfo.getBlockDecl();
1458
1459 CurGD = GD;
1460
1461 CurEHLocation = blockInfo.getBlockExpr()->getEndLoc();
1462
1463 BlockInfo = &blockInfo;
1464
1465 // Arrange for local static and local extern declarations to appear
1466 // to be local to this function as well, in case they're directly
1467 // referenced in a block.
1468 for (const auto &KV : ldm) {
1469 const auto *var = dyn_cast<VarDecl>(Val: KV.first);
1470 if (var && !var->hasLocalStorage())
1471 setAddrOfLocalVar(VD: var, Addr: KV.second);
1472 }
1473
1474 // Begin building the function declaration.
1475
1476 // Build the argument list.
1477 FunctionArgList args;
1478
1479 // The first argument is the block pointer. Just take it as a void*
1480 // and cast it later.
1481 QualType selfTy = getContext().VoidPtrTy;
1482
1483 // For OpenCL passed block pointer can be private AS local variable or
1484 // global AS program scope variable (for the case with and without captures).
1485 // Generic AS is used therefore to be able to accommodate both private and
1486 // generic AS in one implementation.
1487 if (getLangOpts().OpenCL)
1488 selfTy = getContext().getPointerType(T: getContext().getAddrSpaceQualType(
1489 T: getContext().VoidTy, AddressSpace: LangAS::opencl_generic));
1490
1491 const IdentifierInfo *II = &CGM.getContext().Idents.get(Name: ".block_descriptor");
1492
1493 ImplicitParamDecl SelfDecl(getContext(), const_cast<BlockDecl *>(blockDecl),
1494 SourceLocation(), II, selfTy,
1495 ImplicitParamKind::ObjCSelf);
1496 args.push_back(Elt: &SelfDecl);
1497
1498 // Now add the rest of the parameters.
1499 args.append(in_start: blockDecl->param_begin(), in_end: blockDecl->param_end());
1500
1501 // Create the function declaration.
1502 const FunctionProtoType *fnType = blockInfo.getBlockExpr()->getFunctionType();
1503 const CGFunctionInfo &fnInfo =
1504 CGM.getTypes().arrangeBlockFunctionDeclaration(type: fnType, args);
1505 if (CGM.ReturnSlotInterferesWithArgs(FI: fnInfo))
1506 blockInfo.UsesStret = true;
1507
1508 llvm::FunctionType *fnLLVMType = CGM.getTypes().GetFunctionType(Info: fnInfo);
1509
1510 StringRef name = CGM.getBlockMangledName(GD, BD: blockDecl);
1511 llvm::Function *fn = llvm::Function::Create(
1512 Ty: fnLLVMType, Linkage: llvm::GlobalValue::InternalLinkage, N: name, M: &CGM.getModule());
1513 CGM.SetInternalFunctionAttributes(GD: blockDecl, F: fn, FI: fnInfo);
1514
1515 if (BuildGlobalBlock) {
1516 auto GenVoidPtrTy = getContext().getLangOpts().OpenCL
1517 ? CGM.getOpenCLRuntime().getGenericVoidPointerType()
1518 : VoidPtrTy;
1519 buildGlobalBlock(CGM, blockInfo,
1520 blockFn: llvm::ConstantExpr::getPointerCast(C: fn, Ty: GenVoidPtrTy));
1521 }
1522
1523 // Begin generating the function.
1524 StartFunction(GD: blockDecl, RetTy: fnType->getReturnType(), Fn: fn, FnInfo: fnInfo, Args: args,
1525 Loc: blockDecl->getLocation(),
1526 StartLoc: blockInfo.getBlockExpr()->getBody()->getBeginLoc());
1527
1528 // Okay. Undo some of what StartFunction did.
1529
1530 // At -O0 we generate an explicit alloca for the BlockPointer, so the RA
1531 // won't delete the dbg.declare intrinsics for captured variables.
1532 llvm::Value *BlockPointerDbgLoc = BlockPointer;
1533 if (CGM.getCodeGenOpts().OptimizationLevel == 0) {
1534 // Allocate a stack slot for it, so we can point the debugger to it
1535 Address Alloca = CreateTempAlloca(Ty: BlockPointer->getType(),
1536 align: getPointerAlign(),
1537 Name: "block.addr");
1538 // Set the DebugLocation to empty, so the store is recognized as a
1539 // frame setup instruction by llvm::DwarfDebug::beginFunction().
1540 auto NL = ApplyDebugLocation::CreateEmpty(CGF&: *this);
1541 Builder.CreateStore(Val: BlockPointer, Addr: Alloca);
1542 BlockPointerDbgLoc = Alloca.emitRawPointer(CGF&: *this);
1543 }
1544
1545 // If we have a C++ 'this' reference, go ahead and force it into
1546 // existence now.
1547 if (blockDecl->capturesCXXThis()) {
1548 Address addr = Builder.CreateStructGEP(
1549 Addr: LoadBlockStruct(), Index: blockInfo.CXXThisIndex, Name: "block.captured-this");
1550 CXXThisValue = Builder.CreateLoad(Addr: addr, Name: "this");
1551 }
1552
1553 // Also force all the constant captures.
1554 for (const auto &CI : blockDecl->captures()) {
1555 const VarDecl *variable = CI.getVariable();
1556 const CGBlockInfo::Capture &capture = blockInfo.getCapture(var: variable);
1557 if (!capture.isConstant()) continue;
1558
1559 CharUnits align = getContext().getDeclAlign(D: variable);
1560 Address alloca =
1561 CreateMemTemp(T: variable->getType(), Align: align, Name: "block.captured-const");
1562
1563 Builder.CreateStore(Val: capture.getConstant(), Addr: alloca);
1564
1565 setAddrOfLocalVar(VD: variable, Addr: alloca);
1566 }
1567
1568 // Save a spot to insert the debug information for all the DeclRefExprs.
1569 llvm::BasicBlock *entry = Builder.GetInsertBlock();
1570 llvm::BasicBlock::iterator entry_ptr = Builder.GetInsertPoint();
1571 --entry_ptr;
1572
1573 if (IsLambdaConversionToBlock)
1574 EmitLambdaBlockInvokeBody();
1575 else {
1576 PGO->assignRegionCounters(GD: GlobalDecl(blockDecl), Fn: fn);
1577 incrementProfileCounter(S: blockDecl->getBody());
1578 EmitStmt(S: blockDecl->getBody());
1579 }
1580
1581 // Remember where we were...
1582 llvm::BasicBlock *resume = Builder.GetInsertBlock();
1583
1584 // Go back to the entry.
1585 if (entry_ptr->getNextNode())
1586 entry_ptr = entry_ptr->getNextNode()->getIterator();
1587 else
1588 entry_ptr = entry->end();
1589 Builder.SetInsertPoint(TheBB: entry, IP: entry_ptr);
1590
1591 // Emit debug information for all the DeclRefExprs.
1592 // FIXME: also for 'this'
1593 if (CGDebugInfo *DI = getDebugInfo()) {
1594 for (const auto &CI : blockDecl->captures()) {
1595 const VarDecl *variable = CI.getVariable();
1596 DI->EmitLocation(Builder, Loc: variable->getLocation());
1597
1598 if (CGM.getCodeGenOpts().hasReducedDebugInfo()) {
1599 const CGBlockInfo::Capture &capture = blockInfo.getCapture(var: variable);
1600 if (capture.isConstant()) {
1601 auto addr = LocalDeclMap.find(Val: variable)->second;
1602 (void)DI->EmitDeclareOfAutoVariable(
1603 Decl: variable, AI: addr.emitRawPointer(CGF&: *this), Builder);
1604 continue;
1605 }
1606
1607 DI->EmitDeclareOfBlockDeclRefVariable(
1608 variable, storage: BlockPointerDbgLoc, Builder, blockInfo,
1609 InsertPoint: entry_ptr == entry->end() ? nullptr : &*entry_ptr);
1610 }
1611 }
1612 // Recover location if it was changed in the above loop.
1613 DI->EmitLocation(Builder,
1614 Loc: cast<CompoundStmt>(Val: blockDecl->getBody())->getRBracLoc());
1615 }
1616
1617 // And resume where we left off.
1618 if (resume == nullptr)
1619 Builder.ClearInsertionPoint();
1620 else
1621 Builder.SetInsertPoint(resume);
1622
1623 FinishFunction(EndLoc: cast<CompoundStmt>(Val: blockDecl->getBody())->getRBracLoc());
1624
1625 return fn;
1626}
1627
1628static std::pair<BlockCaptureEntityKind, BlockFieldFlags>
1629computeCopyInfoForBlockCapture(const BlockDecl::Capture &CI, QualType T,
1630 const LangOptions &LangOpts) {
1631 if (CI.getCopyExpr()) {
1632 assert(!CI.isByRef());
1633 // don't bother computing flags
1634 return std::make_pair(x: BlockCaptureEntityKind::CXXRecord, y: BlockFieldFlags());
1635 }
1636 BlockFieldFlags Flags;
1637 if (CI.isEscapingByref()) {
1638 Flags = BLOCK_FIELD_IS_BYREF;
1639 if (T.isObjCGCWeak())
1640 Flags |= BLOCK_FIELD_IS_WEAK;
1641 return std::make_pair(x: BlockCaptureEntityKind::BlockObject, y&: Flags);
1642 }
1643
1644 if (T.hasAddressDiscriminatedPointerAuth())
1645 return std::make_pair(
1646 x: BlockCaptureEntityKind::AddressDiscriminatedPointerAuth, y&: Flags);
1647
1648 Flags = BLOCK_FIELD_IS_OBJECT;
1649 bool isBlockPointer = T->isBlockPointerType();
1650 if (isBlockPointer)
1651 Flags = BLOCK_FIELD_IS_BLOCK;
1652
1653 switch (T.isNonTrivialToPrimitiveCopy()) {
1654 case QualType::PCK_Struct:
1655 return std::make_pair(x: BlockCaptureEntityKind::NonTrivialCStruct,
1656 y: BlockFieldFlags());
1657 case QualType::PCK_ARCWeak:
1658 // We need to register __weak direct captures with the runtime.
1659 return std::make_pair(x: BlockCaptureEntityKind::ARCWeak, y&: Flags);
1660 case QualType::PCK_ARCStrong:
1661 // We need to retain the copied value for __strong direct captures.
1662 // If it's a block pointer, we have to copy the block and assign that to
1663 // the destination pointer, so we might as well use _Block_object_assign.
1664 // Otherwise we can avoid that.
1665 return std::make_pair(x: !isBlockPointer ? BlockCaptureEntityKind::ARCStrong
1666 : BlockCaptureEntityKind::BlockObject,
1667 y&: Flags);
1668 case QualType::PCK_PtrAuth:
1669 return std::make_pair(
1670 x: BlockCaptureEntityKind::AddressDiscriminatedPointerAuth,
1671 y: BlockFieldFlags());
1672 case QualType::PCK_Trivial:
1673 case QualType::PCK_VolatileTrivial: {
1674 if (!T->isObjCRetainableType())
1675 // For all other types, the memcpy is fine.
1676 return std::make_pair(x: BlockCaptureEntityKind::None, y: BlockFieldFlags());
1677
1678 // Honor the inert __unsafe_unretained qualifier, which doesn't actually
1679 // make it into the type system.
1680 if (T->isObjCInertUnsafeUnretainedType())
1681 return std::make_pair(x: BlockCaptureEntityKind::None, y: BlockFieldFlags());
1682
1683 // Special rules for ARC captures:
1684 Qualifiers QS = T.getQualifiers();
1685
1686 // Non-ARC captures of retainable pointers are strong and
1687 // therefore require a call to _Block_object_assign.
1688 if (!QS.getObjCLifetime() && !LangOpts.ObjCAutoRefCount)
1689 return std::make_pair(x: BlockCaptureEntityKind::BlockObject, y&: Flags);
1690
1691 // Otherwise the memcpy is fine.
1692 return std::make_pair(x: BlockCaptureEntityKind::None, y: BlockFieldFlags());
1693 }
1694 }
1695 llvm_unreachable("after exhaustive PrimitiveCopyKind switch");
1696}
1697
1698namespace {
1699/// Release a __block variable.
1700struct CallBlockRelease final : EHScopeStack::Cleanup {
1701 Address Addr;
1702 BlockFieldFlags FieldFlags;
1703 bool LoadBlockVarAddr, CanThrow;
1704
1705 CallBlockRelease(Address Addr, BlockFieldFlags Flags, bool LoadValue,
1706 bool CT)
1707 : Addr(Addr), FieldFlags(Flags), LoadBlockVarAddr(LoadValue),
1708 CanThrow(CT) {}
1709
1710 void Emit(CodeGenFunction &CGF, Flags flags) override {
1711 llvm::Value *BlockVarAddr;
1712 if (LoadBlockVarAddr) {
1713 BlockVarAddr = CGF.Builder.CreateLoad(Addr);
1714 } else {
1715 BlockVarAddr = Addr.emitRawPointer(CGF);
1716 }
1717
1718 CGF.BuildBlockRelease(DeclPtr: BlockVarAddr, flags: FieldFlags, CanThrow);
1719 }
1720};
1721} // end anonymous namespace
1722
1723/// Check if \p T is a C++ class that has a destructor that can throw.
1724bool CodeGenFunction::cxxDestructorCanThrow(QualType T) {
1725 if (const auto *RD = T->getAsCXXRecordDecl())
1726 if (const CXXDestructorDecl *DD = RD->getDestructor())
1727 return DD->getType()->castAs<FunctionProtoType>()->canThrow();
1728 return false;
1729}
1730
1731// Return a string that has the information about a capture.
1732static std::string getBlockCaptureStr(const CGBlockInfo::Capture &Cap,
1733 CaptureStrKind StrKind,
1734 CharUnits BlockAlignment,
1735 CodeGenModule &CGM) {
1736 std::string Str;
1737 ASTContext &Ctx = CGM.getContext();
1738 const BlockDecl::Capture &CI = *Cap.Cap;
1739 QualType CaptureTy = CI.getVariable()->getType();
1740
1741 BlockCaptureEntityKind Kind;
1742 BlockFieldFlags Flags;
1743
1744 // CaptureStrKind::Merged should be passed only when the operations and the
1745 // flags are the same for copy and dispose.
1746 assert((StrKind != CaptureStrKind::Merged ||
1747 (Cap.CopyKind == Cap.DisposeKind &&
1748 Cap.CopyFlags == Cap.DisposeFlags)) &&
1749 "different operations and flags");
1750
1751 if (StrKind == CaptureStrKind::DisposeHelper) {
1752 Kind = Cap.DisposeKind;
1753 Flags = Cap.DisposeFlags;
1754 } else {
1755 Kind = Cap.CopyKind;
1756 Flags = Cap.CopyFlags;
1757 }
1758
1759 switch (Kind) {
1760 case BlockCaptureEntityKind::CXXRecord: {
1761 Str += "c";
1762 SmallString<256> TyStr;
1763 llvm::raw_svector_ostream Out(TyStr);
1764 CGM.getCXXABI().getMangleContext().mangleCanonicalTypeName(T: CaptureTy, Out);
1765 Str += llvm::to_string(Value: TyStr.size()) + TyStr.c_str();
1766 break;
1767 }
1768 case BlockCaptureEntityKind::ARCWeak:
1769 Str += "w";
1770 break;
1771 case BlockCaptureEntityKind::ARCStrong:
1772 Str += "s";
1773 break;
1774 case BlockCaptureEntityKind::AddressDiscriminatedPointerAuth: {
1775 auto PtrAuth = CaptureTy.getPointerAuth();
1776 assert(PtrAuth && PtrAuth.isAddressDiscriminated());
1777 Str += "p" + llvm::to_string(Value: PtrAuth.getKey()) + "d" +
1778 llvm::to_string(Value: PtrAuth.getExtraDiscriminator());
1779 break;
1780 }
1781 case BlockCaptureEntityKind::BlockObject: {
1782 const VarDecl *Var = CI.getVariable();
1783 unsigned F = Flags.getBitMask();
1784 if (F & BLOCK_FIELD_IS_BYREF) {
1785 Str += "r";
1786 if (F & BLOCK_FIELD_IS_WEAK)
1787 Str += "w";
1788 else {
1789 // If CaptureStrKind::Merged is passed, check both the copy expression
1790 // and the destructor.
1791 if (StrKind != CaptureStrKind::DisposeHelper) {
1792 if (Ctx.getBlockVarCopyInit(VD: Var).canThrow())
1793 Str += "c";
1794 }
1795 if (StrKind != CaptureStrKind::CopyHelper) {
1796 if (CodeGenFunction::cxxDestructorCanThrow(T: CaptureTy))
1797 Str += "d";
1798 }
1799 }
1800 } else {
1801 assert((F & BLOCK_FIELD_IS_OBJECT) && "unexpected flag value");
1802 if (F == BLOCK_FIELD_IS_BLOCK)
1803 Str += "b";
1804 else
1805 Str += "o";
1806 }
1807 break;
1808 }
1809 case BlockCaptureEntityKind::NonTrivialCStruct: {
1810 bool IsVolatile = CaptureTy.isVolatileQualified();
1811 CharUnits Alignment = BlockAlignment.alignmentAtOffset(offset: Cap.getOffset());
1812
1813 Str += "n";
1814 std::string FuncStr;
1815 if (StrKind == CaptureStrKind::DisposeHelper)
1816 FuncStr = CodeGenFunction::getNonTrivialDestructorStr(
1817 QT: CaptureTy, Alignment, IsVolatile, Ctx);
1818 else
1819 // If CaptureStrKind::Merged is passed, use the copy constructor string.
1820 // It has all the information that the destructor string has.
1821 FuncStr = CodeGenFunction::getNonTrivialCopyConstructorStr(
1822 QT: CaptureTy, Alignment, IsVolatile, Ctx);
1823 // The underscore is necessary here because non-trivial copy constructor
1824 // and destructor strings can start with a number.
1825 Str += llvm::to_string(Value: FuncStr.size()) + "_" + FuncStr;
1826 break;
1827 }
1828 case BlockCaptureEntityKind::None:
1829 break;
1830 }
1831
1832 return Str;
1833}
1834
1835static std::string getCopyDestroyHelperFuncName(
1836 const SmallVectorImpl<CGBlockInfo::Capture> &Captures,
1837 CharUnits BlockAlignment, CaptureStrKind StrKind, CodeGenModule &CGM) {
1838 assert((StrKind == CaptureStrKind::CopyHelper ||
1839 StrKind == CaptureStrKind::DisposeHelper) &&
1840 "unexpected CaptureStrKind");
1841 std::string Name = StrKind == CaptureStrKind::CopyHelper
1842 ? "__copy_helper_block_"
1843 : "__destroy_helper_block_";
1844 if (CGM.getLangOpts().Exceptions)
1845 Name += "e";
1846 if (CGM.getCodeGenOpts().ObjCAutoRefCountExceptions)
1847 Name += "a";
1848 Name += llvm::to_string(Value: BlockAlignment.getQuantity()) + "_";
1849
1850 for (auto &Cap : Captures) {
1851 if (Cap.isConstantOrTrivial())
1852 continue;
1853 Name += llvm::to_string(Value: Cap.getOffset().getQuantity());
1854 Name += getBlockCaptureStr(Cap, StrKind, BlockAlignment, CGM);
1855 }
1856
1857 return Name;
1858}
1859
1860static void pushCaptureCleanup(BlockCaptureEntityKind CaptureKind,
1861 Address Field, QualType CaptureType,
1862 BlockFieldFlags Flags, bool ForCopyHelper,
1863 VarDecl *Var, CodeGenFunction &CGF) {
1864 bool EHOnly = ForCopyHelper;
1865
1866 switch (CaptureKind) {
1867 case BlockCaptureEntityKind::CXXRecord:
1868 case BlockCaptureEntityKind::ARCWeak:
1869 case BlockCaptureEntityKind::NonTrivialCStruct:
1870 case BlockCaptureEntityKind::ARCStrong: {
1871 if (CaptureType.isDestructedType() &&
1872 (!EHOnly || CGF.needsEHCleanup(kind: CaptureType.isDestructedType()))) {
1873 CodeGenFunction::Destroyer *Destroyer =
1874 CaptureKind == BlockCaptureEntityKind::ARCStrong
1875 ? CodeGenFunction::destroyARCStrongImprecise
1876 : CGF.getDestroyer(destructionKind: CaptureType.isDestructedType());
1877 CleanupKind Kind =
1878 EHOnly ? EHCleanup
1879 : CGF.getCleanupKind(kind: CaptureType.isDestructedType());
1880 CGF.pushDestroy(kind: Kind, addr: Field, type: CaptureType, destroyer: Destroyer, useEHCleanupForArray: Kind & EHCleanup);
1881 }
1882 break;
1883 }
1884 case BlockCaptureEntityKind::BlockObject: {
1885 if (!EHOnly || CGF.getLangOpts().Exceptions) {
1886 CleanupKind Kind = EHOnly ? EHCleanup : NormalAndEHCleanup;
1887 // Calls to _Block_object_dispose along the EH path in the copy helper
1888 // function don't throw as newly-copied __block variables always have a
1889 // reference count of 2.
1890 bool CanThrow =
1891 !ForCopyHelper && CGF.cxxDestructorCanThrow(T: CaptureType);
1892 CGF.enterByrefCleanup(Kind, Addr: Field, Flags, /*LoadBlockVarAddr*/ true,
1893 CanThrow);
1894 }
1895 break;
1896 }
1897 case BlockCaptureEntityKind::AddressDiscriminatedPointerAuth:
1898 case BlockCaptureEntityKind::None:
1899 break;
1900 }
1901}
1902
1903static void setBlockHelperAttributesVisibility(bool CapturesNonExternalType,
1904 llvm::Function *Fn,
1905 const CGFunctionInfo &FI,
1906 CodeGenModule &CGM) {
1907 if (CapturesNonExternalType) {
1908 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI);
1909 } else {
1910 Fn->setVisibility(llvm::GlobalValue::HiddenVisibility);
1911 Fn->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
1912 CGM.SetLLVMFunctionAttributes(GD: GlobalDecl(), Info: FI, F: Fn, /*IsThunk=*/false);
1913 CGM.SetLLVMFunctionAttributesForDefinition(D: nullptr, F: Fn);
1914 }
1915}
1916/// Generate the copy-helper function for a block closure object:
1917/// static void block_copy_helper(block_t *dst, block_t *src);
1918/// The runtime will have previously initialized 'dst' by doing a
1919/// bit-copy of 'src'.
1920///
1921/// Note that this copies an entire block closure object to the heap;
1922/// it should not be confused with a 'byref copy helper', which moves
1923/// the contents of an individual __block variable to the heap.
1924llvm::Constant *
1925CodeGenFunction::GenerateCopyHelperFunction(const CGBlockInfo &blockInfo) {
1926 std::string FuncName = getCopyDestroyHelperFuncName(
1927 Captures: blockInfo.SortedCaptures, BlockAlignment: blockInfo.BlockAlign,
1928 StrKind: CaptureStrKind::CopyHelper, CGM);
1929
1930 if (llvm::GlobalValue *Func = CGM.getModule().getNamedValue(Name: FuncName))
1931 return Func;
1932
1933 ASTContext &C = getContext();
1934
1935 QualType ReturnTy = C.VoidTy;
1936
1937 FunctionArgList args;
1938 ImplicitParamDecl DstDecl(C, C.VoidPtrTy, ImplicitParamKind::Other);
1939 args.push_back(Elt: &DstDecl);
1940 ImplicitParamDecl SrcDecl(C, C.VoidPtrTy, ImplicitParamKind::Other);
1941 args.push_back(Elt: &SrcDecl);
1942
1943 const CGFunctionInfo &FI =
1944 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: ReturnTy, args);
1945
1946 // FIXME: it would be nice if these were mergeable with things with
1947 // identical semantics.
1948 llvm::FunctionType *LTy = CGM.getTypes().GetFunctionType(Info: FI);
1949
1950 llvm::Function *Fn =
1951 llvm::Function::Create(Ty: LTy, Linkage: llvm::GlobalValue::LinkOnceODRLinkage,
1952 N: FuncName, M: &CGM.getModule());
1953 if (CGM.supportsCOMDAT())
1954 Fn->setComdat(CGM.getModule().getOrInsertComdat(Name: FuncName));
1955
1956 SmallVector<QualType, 2> ArgTys;
1957 ArgTys.push_back(Elt: C.VoidPtrTy);
1958 ArgTys.push_back(Elt: C.VoidPtrTy);
1959
1960 setBlockHelperAttributesVisibility(CapturesNonExternalType: blockInfo.CapturesNonExternalType, Fn, FI,
1961 CGM);
1962 StartFunction(GD: GlobalDecl(), RetTy: ReturnTy, Fn, FnInfo: FI, Args: args);
1963 auto AL = ApplyDebugLocation::CreateArtificial(CGF&: *this);
1964
1965 Address src = GetAddrOfLocalVar(VD: &SrcDecl);
1966 src = Address(Builder.CreateLoad(Addr: src), blockInfo.StructureType,
1967 blockInfo.BlockAlign);
1968
1969 Address dst = GetAddrOfLocalVar(VD: &DstDecl);
1970 dst = Address(Builder.CreateLoad(Addr: dst), blockInfo.StructureType,
1971 blockInfo.BlockAlign);
1972
1973 for (auto &capture : blockInfo.SortedCaptures) {
1974 if (capture.isConstantOrTrivial())
1975 continue;
1976
1977 const BlockDecl::Capture &CI = *capture.Cap;
1978 QualType captureType = CI.getVariable()->getType();
1979 BlockFieldFlags flags = capture.CopyFlags;
1980
1981 unsigned index = capture.getIndex();
1982 Address srcField = Builder.CreateStructGEP(Addr: src, Index: index);
1983 Address dstField = Builder.CreateStructGEP(Addr: dst, Index: index);
1984
1985 switch (capture.CopyKind) {
1986 case BlockCaptureEntityKind::CXXRecord:
1987 // If there's an explicit copy expression, we do that.
1988 assert(CI.getCopyExpr() && "copy expression for variable is missing");
1989 EmitSynthesizedCXXCopyCtor(Dest: dstField, Src: srcField, Exp: CI.getCopyExpr());
1990 break;
1991 case BlockCaptureEntityKind::ARCWeak:
1992 EmitARCCopyWeak(dst: dstField, src: srcField);
1993 break;
1994 case BlockCaptureEntityKind::AddressDiscriminatedPointerAuth: {
1995 QualType Type = CI.getVariable()->getType();
1996 PointerAuthQualifier PointerAuth = Type.getPointerAuth();
1997 assert(PointerAuth && PointerAuth.isAddressDiscriminated());
1998 EmitPointerAuthCopy(Qualifier: PointerAuth, Type, DestField: dstField, SrcField: srcField);
1999 // We don't need to push cleanups for ptrauth types.
2000 continue;
2001 }
2002 case BlockCaptureEntityKind::NonTrivialCStruct: {
2003 // If this is a C struct that requires non-trivial copy construction,
2004 // emit a call to its copy constructor.
2005 QualType varType = CI.getVariable()->getType();
2006 callCStructCopyConstructor(Dst: MakeAddrLValue(Addr: dstField, T: varType),
2007 Src: MakeAddrLValue(Addr: srcField, T: varType));
2008 break;
2009 }
2010 case BlockCaptureEntityKind::ARCStrong: {
2011 llvm::Value *srcValue = Builder.CreateLoad(Addr: srcField, Name: "blockcopy.src");
2012 // At -O0, store null into the destination field (so that the
2013 // storeStrong doesn't over-release) and then call storeStrong.
2014 // This is a workaround to not having an initStrong call.
2015 if (CGM.getCodeGenOpts().OptimizationLevel == 0) {
2016 auto *ty = cast<llvm::PointerType>(Val: srcValue->getType());
2017 llvm::Value *null = llvm::ConstantPointerNull::get(T: ty);
2018 Builder.CreateStore(Val: null, Addr: dstField);
2019 EmitARCStoreStrongCall(addr: dstField, value: srcValue, resultIgnored: true);
2020
2021 // With optimization enabled, take advantage of the fact that
2022 // the blocks runtime guarantees a memcpy of the block data, and
2023 // just emit a retain of the src field.
2024 } else {
2025 EmitARCRetainNonBlock(value: srcValue);
2026
2027 // Unless EH cleanup is required, we don't need this anymore, so kill
2028 // it. It's not quite worth the annoyance to avoid creating it in the
2029 // first place.
2030 if (!needsEHCleanup(kind: captureType.isDestructedType()))
2031 if (auto *I = cast_or_null<llvm::Instruction>(
2032 Val: dstField.getPointerIfNotSigned()))
2033 I->eraseFromParent();
2034 }
2035 break;
2036 }
2037 case BlockCaptureEntityKind::BlockObject: {
2038 llvm::Value *srcValue = Builder.CreateLoad(Addr: srcField, Name: "blockcopy.src");
2039 llvm::Value *dstAddr = dstField.emitRawPointer(CGF&: *this);
2040 llvm::Value *args[] = {
2041 dstAddr, srcValue, llvm::ConstantInt::get(Ty: Int32Ty, V: flags.getBitMask())
2042 };
2043
2044 if (CI.isByRef() && C.getBlockVarCopyInit(VD: CI.getVariable()).canThrow())
2045 EmitRuntimeCallOrInvoke(callee: CGM.getBlockObjectAssign(), args);
2046 else
2047 EmitNounwindRuntimeCall(callee: CGM.getBlockObjectAssign(), args);
2048 break;
2049 }
2050 case BlockCaptureEntityKind::None:
2051 continue;
2052 }
2053
2054 // Ensure that we destroy the copied object if an exception is thrown later
2055 // in the helper function.
2056 pushCaptureCleanup(CaptureKind: capture.CopyKind, Field: dstField, CaptureType: captureType, Flags: flags,
2057 /*ForCopyHelper*/ true, Var: CI.getVariable(), CGF&: *this);
2058 }
2059
2060 FinishFunction();
2061
2062 return Fn;
2063}
2064
2065static BlockFieldFlags
2066getBlockFieldFlagsForObjCObjectPointer(const BlockDecl::Capture &CI,
2067 QualType T) {
2068 BlockFieldFlags Flags = BLOCK_FIELD_IS_OBJECT;
2069 if (T->isBlockPointerType())
2070 Flags = BLOCK_FIELD_IS_BLOCK;
2071 return Flags;
2072}
2073
2074static std::pair<BlockCaptureEntityKind, BlockFieldFlags>
2075computeDestroyInfoForBlockCapture(const BlockDecl::Capture &CI, QualType T,
2076 const LangOptions &LangOpts) {
2077 if (CI.isEscapingByref()) {
2078 BlockFieldFlags Flags = BLOCK_FIELD_IS_BYREF;
2079 if (T.isObjCGCWeak())
2080 Flags |= BLOCK_FIELD_IS_WEAK;
2081 return std::make_pair(x: BlockCaptureEntityKind::BlockObject, y&: Flags);
2082 }
2083
2084 switch (T.isDestructedType()) {
2085 case QualType::DK_cxx_destructor:
2086 return std::make_pair(x: BlockCaptureEntityKind::CXXRecord, y: BlockFieldFlags());
2087 case QualType::DK_objc_strong_lifetime:
2088 // Use objc_storeStrong for __strong direct captures; the
2089 // dynamic tools really like it when we do this.
2090 return std::make_pair(x: BlockCaptureEntityKind::ARCStrong,
2091 y: getBlockFieldFlagsForObjCObjectPointer(CI, T));
2092 case QualType::DK_objc_weak_lifetime:
2093 // Support __weak direct captures.
2094 return std::make_pair(x: BlockCaptureEntityKind::ARCWeak,
2095 y: getBlockFieldFlagsForObjCObjectPointer(CI, T));
2096 case QualType::DK_nontrivial_c_struct:
2097 return std::make_pair(x: BlockCaptureEntityKind::NonTrivialCStruct,
2098 y: BlockFieldFlags());
2099 case QualType::DK_none: {
2100 // Non-ARC captures are strong, and we need to use _Block_object_dispose.
2101 // But honor the inert __unsafe_unretained qualifier, which doesn't actually
2102 // make it into the type system.
2103 if (T->isObjCRetainableType() && !T.getQualifiers().hasObjCLifetime() &&
2104 !LangOpts.ObjCAutoRefCount && !T->isObjCInertUnsafeUnretainedType())
2105 return std::make_pair(x: BlockCaptureEntityKind::BlockObject,
2106 y: getBlockFieldFlagsForObjCObjectPointer(CI, T));
2107 // Otherwise, we have nothing to do.
2108 return std::make_pair(x: BlockCaptureEntityKind::None, y: BlockFieldFlags());
2109 }
2110 }
2111 llvm_unreachable("after exhaustive DestructionKind switch");
2112}
2113
2114/// Generate the destroy-helper function for a block closure object:
2115/// static void block_destroy_helper(block_t *theBlock);
2116///
2117/// Note that this destroys a heap-allocated block closure object;
2118/// it should not be confused with a 'byref destroy helper', which
2119/// destroys the heap-allocated contents of an individual __block
2120/// variable.
2121llvm::Constant *
2122CodeGenFunction::GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo) {
2123 std::string FuncName = getCopyDestroyHelperFuncName(
2124 Captures: blockInfo.SortedCaptures, BlockAlignment: blockInfo.BlockAlign,
2125 StrKind: CaptureStrKind::DisposeHelper, CGM);
2126
2127 if (llvm::GlobalValue *Func = CGM.getModule().getNamedValue(Name: FuncName))
2128 return Func;
2129
2130 ASTContext &C = getContext();
2131
2132 QualType ReturnTy = C.VoidTy;
2133
2134 FunctionArgList args;
2135 ImplicitParamDecl SrcDecl(C, C.VoidPtrTy, ImplicitParamKind::Other);
2136 args.push_back(Elt: &SrcDecl);
2137
2138 const CGFunctionInfo &FI =
2139 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: ReturnTy, args);
2140
2141 // FIXME: We'd like to put these into a mergable by content, with
2142 // internal linkage.
2143 llvm::FunctionType *LTy = CGM.getTypes().GetFunctionType(Info: FI);
2144
2145 llvm::Function *Fn =
2146 llvm::Function::Create(Ty: LTy, Linkage: llvm::GlobalValue::LinkOnceODRLinkage,
2147 N: FuncName, M: &CGM.getModule());
2148 if (CGM.supportsCOMDAT())
2149 Fn->setComdat(CGM.getModule().getOrInsertComdat(Name: FuncName));
2150
2151 SmallVector<QualType, 1> ArgTys;
2152 ArgTys.push_back(Elt: C.VoidPtrTy);
2153
2154 setBlockHelperAttributesVisibility(CapturesNonExternalType: blockInfo.CapturesNonExternalType, Fn, FI,
2155 CGM);
2156 StartFunction(GD: GlobalDecl(), RetTy: ReturnTy, Fn, FnInfo: FI, Args: args);
2157 markAsIgnoreThreadCheckingAtRuntime(Fn);
2158
2159 auto AL = ApplyDebugLocation::CreateArtificial(CGF&: *this);
2160
2161 Address src = GetAddrOfLocalVar(VD: &SrcDecl);
2162 src = Address(Builder.CreateLoad(Addr: src), blockInfo.StructureType,
2163 blockInfo.BlockAlign);
2164
2165 CodeGenFunction::RunCleanupsScope cleanups(*this);
2166
2167 for (auto &capture : blockInfo.SortedCaptures) {
2168 if (capture.isConstantOrTrivial())
2169 continue;
2170
2171 const BlockDecl::Capture &CI = *capture.Cap;
2172 BlockFieldFlags flags = capture.DisposeFlags;
2173
2174 Address srcField = Builder.CreateStructGEP(Addr: src, Index: capture.getIndex());
2175
2176 pushCaptureCleanup(CaptureKind: capture.DisposeKind, Field: srcField,
2177 CaptureType: CI.getVariable()->getType(), Flags: flags,
2178 /*ForCopyHelper*/ false, Var: CI.getVariable(), CGF&: *this);
2179 }
2180
2181 cleanups.ForceCleanup();
2182
2183 FinishFunction();
2184
2185 return Fn;
2186}
2187
2188namespace {
2189
2190/// Emits the copy/dispose helper functions for a __block object of id type.
2191class ObjectByrefHelpers final : public BlockByrefHelpers {
2192 BlockFieldFlags Flags;
2193
2194public:
2195 ObjectByrefHelpers(CharUnits alignment, BlockFieldFlags flags)
2196 : BlockByrefHelpers(alignment), Flags(flags) {}
2197
2198 void emitCopy(CodeGenFunction &CGF, Address destField,
2199 Address srcField) override {
2200 destField = destField.withElementType(ElemTy: CGF.Int8Ty);
2201
2202 srcField = srcField.withElementType(ElemTy: CGF.Int8PtrTy);
2203 llvm::Value *srcValue = CGF.Builder.CreateLoad(Addr: srcField);
2204
2205 unsigned flags = (Flags | BLOCK_BYREF_CALLER).getBitMask();
2206
2207 llvm::Value *flagsVal = llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: flags);
2208 llvm::FunctionCallee fn = CGF.CGM.getBlockObjectAssign();
2209
2210 llvm::Value *args[] = {destField.emitRawPointer(CGF), srcValue, flagsVal};
2211 CGF.EmitNounwindRuntimeCall(callee: fn, args);
2212 }
2213
2214 void emitDispose(CodeGenFunction &CGF, Address field) override {
2215 field = field.withElementType(ElemTy: CGF.Int8PtrTy);
2216 llvm::Value *value = CGF.Builder.CreateLoad(Addr: field);
2217
2218 CGF.BuildBlockRelease(DeclPtr: value, flags: Flags | BLOCK_BYREF_CALLER, CanThrow: false);
2219 }
2220
2221 void profileImpl(llvm::FoldingSetNodeID &id) const override {
2222 id.AddInteger(I: Flags.getBitMask());
2223 }
2224};
2225
2226/// Emits the copy/dispose helpers for an ARC __block __weak variable.
2227class ARCWeakByrefHelpers final : public BlockByrefHelpers {
2228public:
2229 ARCWeakByrefHelpers(CharUnits alignment) : BlockByrefHelpers(alignment) {}
2230
2231 void emitCopy(CodeGenFunction &CGF, Address destField,
2232 Address srcField) override {
2233 CGF.EmitARCMoveWeak(dst: destField, src: srcField);
2234 }
2235
2236 void emitDispose(CodeGenFunction &CGF, Address field) override {
2237 CGF.EmitARCDestroyWeak(addr: field);
2238 }
2239
2240 void profileImpl(llvm::FoldingSetNodeID &id) const override {
2241 // 0 is distinguishable from all pointers and byref flags
2242 id.AddInteger(I: 0);
2243 }
2244};
2245
2246/// Emits the copy/dispose helpers for an ARC __block __strong variable
2247/// that's not of block-pointer type.
2248class ARCStrongByrefHelpers final : public BlockByrefHelpers {
2249public:
2250 ARCStrongByrefHelpers(CharUnits alignment) : BlockByrefHelpers(alignment) {}
2251
2252 void emitCopy(CodeGenFunction &CGF, Address destField,
2253 Address srcField) override {
2254 // Do a "move" by copying the value and then zeroing out the old
2255 // variable.
2256
2257 llvm::Value *value = CGF.Builder.CreateLoad(Addr: srcField);
2258
2259 llvm::Value *null =
2260 llvm::ConstantPointerNull::get(T: cast<llvm::PointerType>(Val: value->getType()));
2261
2262 if (CGF.CGM.getCodeGenOpts().OptimizationLevel == 0) {
2263 CGF.Builder.CreateStore(Val: null, Addr: destField);
2264 CGF.EmitARCStoreStrongCall(addr: destField, value, /*ignored*/ resultIgnored: true);
2265 CGF.EmitARCStoreStrongCall(addr: srcField, value: null, /*ignored*/ resultIgnored: true);
2266 return;
2267 }
2268 CGF.Builder.CreateStore(Val: value, Addr: destField);
2269 CGF.Builder.CreateStore(Val: null, Addr: srcField);
2270 }
2271
2272 void emitDispose(CodeGenFunction &CGF, Address field) override {
2273 CGF.EmitARCDestroyStrong(addr: field, precise: ARCImpreciseLifetime);
2274 }
2275
2276 void profileImpl(llvm::FoldingSetNodeID &id) const override {
2277 // 1 is distinguishable from all pointers and byref flags
2278 id.AddInteger(I: 1);
2279 }
2280};
2281
2282/// Emits the copy/dispose helpers for an ARC __block __strong
2283/// variable that's of block-pointer type.
2284class ARCStrongBlockByrefHelpers final : public BlockByrefHelpers {
2285public:
2286 ARCStrongBlockByrefHelpers(CharUnits alignment)
2287 : BlockByrefHelpers(alignment) {}
2288
2289 void emitCopy(CodeGenFunction &CGF, Address destField,
2290 Address srcField) override {
2291 // Do the copy with objc_retainBlock; that's all that
2292 // _Block_object_assign would do anyway, and we'd have to pass the
2293 // right arguments to make sure it doesn't get no-op'ed.
2294 llvm::Value *oldValue = CGF.Builder.CreateLoad(Addr: srcField);
2295 llvm::Value *copy = CGF.EmitARCRetainBlock(value: oldValue, /*mandatory*/ true);
2296 CGF.Builder.CreateStore(Val: copy, Addr: destField);
2297 }
2298
2299 void emitDispose(CodeGenFunction &CGF, Address field) override {
2300 CGF.EmitARCDestroyStrong(addr: field, precise: ARCImpreciseLifetime);
2301 }
2302
2303 void profileImpl(llvm::FoldingSetNodeID &id) const override {
2304 // 2 is distinguishable from all pointers and byref flags
2305 id.AddInteger(I: 2);
2306 }
2307};
2308
2309/// Emits the copy/dispose helpers for a __block variable with a
2310/// nontrivial copy constructor or destructor.
2311class CXXByrefHelpers final : public BlockByrefHelpers {
2312 QualType VarType;
2313 const Expr *CopyExpr;
2314
2315public:
2316 CXXByrefHelpers(CharUnits alignment, QualType type,
2317 const Expr *copyExpr)
2318 : BlockByrefHelpers(alignment), VarType(type), CopyExpr(copyExpr) {}
2319
2320 bool needsCopy() const override { return CopyExpr != nullptr; }
2321 void emitCopy(CodeGenFunction &CGF, Address destField,
2322 Address srcField) override {
2323 if (!CopyExpr) return;
2324 CGF.EmitSynthesizedCXXCopyCtor(Dest: destField, Src: srcField, Exp: CopyExpr);
2325 }
2326
2327 void emitDispose(CodeGenFunction &CGF, Address field) override {
2328 EHScopeStack::stable_iterator cleanupDepth = CGF.EHStack.stable_begin();
2329 CGF.PushDestructorCleanup(T: VarType, Addr: field);
2330 CGF.PopCleanupBlocks(OldCleanupStackSize: cleanupDepth);
2331 }
2332
2333 void profileImpl(llvm::FoldingSetNodeID &id) const override {
2334 id.AddPointer(Ptr: VarType.getCanonicalType().getAsOpaquePtr());
2335 }
2336};
2337
2338/// Emits the copy/dispose helpers for a __block variable with
2339/// address-discriminated pointer authentication.
2340class AddressDiscriminatedByrefHelpers final : public BlockByrefHelpers {
2341 QualType VarType;
2342
2343public:
2344 AddressDiscriminatedByrefHelpers(CharUnits Alignment, QualType Type)
2345 : BlockByrefHelpers(Alignment), VarType(Type) {
2346 assert(Type.hasAddressDiscriminatedPointerAuth());
2347 }
2348
2349 void emitCopy(CodeGenFunction &CGF, Address DestField,
2350 Address SrcField) override {
2351 CGF.EmitPointerAuthCopy(Qualifier: VarType.getPointerAuth(), Type: VarType, DestField,
2352 SrcField);
2353 }
2354
2355 bool needsDispose() const override { return false; }
2356 void emitDispose(CodeGenFunction &CGF, Address Field) override {
2357 llvm_unreachable("should never be called");
2358 }
2359
2360 void profileImpl(llvm::FoldingSetNodeID &ID) const override {
2361 ID.AddPointer(Ptr: VarType.getCanonicalType().getAsOpaquePtr());
2362 }
2363};
2364
2365/// Emits the copy/dispose helpers for a __block variable that is a non-trivial
2366/// C struct.
2367class NonTrivialCStructByrefHelpers final : public BlockByrefHelpers {
2368 QualType VarType;
2369
2370public:
2371 NonTrivialCStructByrefHelpers(CharUnits alignment, QualType type)
2372 : BlockByrefHelpers(alignment), VarType(type) {}
2373
2374 void emitCopy(CodeGenFunction &CGF, Address destField,
2375 Address srcField) override {
2376 CGF.callCStructMoveConstructor(Dst: CGF.MakeAddrLValue(Addr: destField, T: VarType),
2377 Src: CGF.MakeAddrLValue(Addr: srcField, T: VarType));
2378 }
2379
2380 bool needsDispose() const override {
2381 return VarType.isDestructedType();
2382 }
2383
2384 void emitDispose(CodeGenFunction &CGF, Address field) override {
2385 EHScopeStack::stable_iterator cleanupDepth = CGF.EHStack.stable_begin();
2386 CGF.pushDestroy(dtorKind: VarType.isDestructedType(), addr: field, type: VarType);
2387 CGF.PopCleanupBlocks(OldCleanupStackSize: cleanupDepth);
2388 }
2389
2390 void profileImpl(llvm::FoldingSetNodeID &id) const override {
2391 id.AddPointer(Ptr: VarType.getCanonicalType().getAsOpaquePtr());
2392 }
2393};
2394} // end anonymous namespace
2395
2396static llvm::Constant *
2397generateByrefCopyHelper(CodeGenFunction &CGF, const BlockByrefInfo &byrefInfo,
2398 BlockByrefHelpers &generator) {
2399 ASTContext &Context = CGF.getContext();
2400
2401 QualType ReturnTy = Context.VoidTy;
2402
2403 FunctionArgList args;
2404 ImplicitParamDecl Dst(Context, Context.VoidPtrTy, ImplicitParamKind::Other);
2405 args.push_back(Elt: &Dst);
2406
2407 ImplicitParamDecl Src(Context, Context.VoidPtrTy, ImplicitParamKind::Other);
2408 args.push_back(Elt: &Src);
2409
2410 const CGFunctionInfo &FI =
2411 CGF.CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: ReturnTy, args);
2412
2413 llvm::FunctionType *LTy = CGF.CGM.getTypes().GetFunctionType(Info: FI);
2414
2415 // FIXME: We'd like to put these into a mergable by content, with
2416 // internal linkage.
2417 llvm::Function *Fn =
2418 llvm::Function::Create(Ty: LTy, Linkage: llvm::GlobalValue::InternalLinkage,
2419 N: "__Block_byref_object_copy_", M: &CGF.CGM.getModule());
2420
2421 SmallVector<QualType, 2> ArgTys;
2422 ArgTys.push_back(Elt: Context.VoidPtrTy);
2423 ArgTys.push_back(Elt: Context.VoidPtrTy);
2424
2425 CGF.CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI);
2426
2427 CGF.StartFunction(GD: GlobalDecl(), RetTy: ReturnTy, Fn, FnInfo: FI, Args: args);
2428 // Create a scope with an artificial location for the body of this function.
2429 auto AL = ApplyDebugLocation::CreateArtificial(CGF);
2430
2431 if (generator.needsCopy()) {
2432 // dst->x
2433 Address destField = CGF.GetAddrOfLocalVar(VD: &Dst);
2434 destField = Address(CGF.Builder.CreateLoad(Addr: destField), byrefInfo.Type,
2435 byrefInfo.ByrefAlignment);
2436 destField =
2437 CGF.emitBlockByrefAddress(baseAddr: destField, info: byrefInfo, followForward: false, name: "dest-object");
2438
2439 // src->x
2440 Address srcField = CGF.GetAddrOfLocalVar(VD: &Src);
2441 srcField = Address(CGF.Builder.CreateLoad(Addr: srcField), byrefInfo.Type,
2442 byrefInfo.ByrefAlignment);
2443 srcField =
2444 CGF.emitBlockByrefAddress(baseAddr: srcField, info: byrefInfo, followForward: false, name: "src-object");
2445
2446 generator.emitCopy(CGF, dest: destField, src: srcField);
2447 }
2448
2449 CGF.FinishFunction();
2450
2451 return Fn;
2452}
2453
2454/// Build the copy helper for a __block variable.
2455static llvm::Constant *buildByrefCopyHelper(CodeGenModule &CGM,
2456 const BlockByrefInfo &byrefInfo,
2457 BlockByrefHelpers &generator) {
2458 CodeGenFunction CGF(CGM);
2459 return generateByrefCopyHelper(CGF, byrefInfo, generator);
2460}
2461
2462/// Generate code for a __block variable's dispose helper.
2463static llvm::Constant *
2464generateByrefDisposeHelper(CodeGenFunction &CGF,
2465 const BlockByrefInfo &byrefInfo,
2466 BlockByrefHelpers &generator) {
2467 ASTContext &Context = CGF.getContext();
2468 QualType R = Context.VoidTy;
2469
2470 FunctionArgList args;
2471 ImplicitParamDecl Src(CGF.getContext(), Context.VoidPtrTy,
2472 ImplicitParamKind::Other);
2473 args.push_back(Elt: &Src);
2474
2475 const CGFunctionInfo &FI =
2476 CGF.CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: R, args);
2477
2478 llvm::FunctionType *LTy = CGF.CGM.getTypes().GetFunctionType(Info: FI);
2479
2480 // FIXME: We'd like to put these into a mergable by content, with
2481 // internal linkage.
2482 llvm::Function *Fn =
2483 llvm::Function::Create(Ty: LTy, Linkage: llvm::GlobalValue::InternalLinkage,
2484 N: "__Block_byref_object_dispose_",
2485 M: &CGF.CGM.getModule());
2486
2487 SmallVector<QualType, 1> ArgTys;
2488 ArgTys.push_back(Elt: Context.VoidPtrTy);
2489
2490 CGF.CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI);
2491
2492 CGF.StartFunction(GD: GlobalDecl(), RetTy: R, Fn, FnInfo: FI, Args: args);
2493 // Create a scope with an artificial location for the body of this function.
2494 auto AL = ApplyDebugLocation::CreateArtificial(CGF);
2495
2496 if (generator.needsDispose()) {
2497 Address addr = CGF.GetAddrOfLocalVar(VD: &Src);
2498 addr = Address(CGF.Builder.CreateLoad(Addr: addr), byrefInfo.Type,
2499 byrefInfo.ByrefAlignment);
2500 addr = CGF.emitBlockByrefAddress(baseAddr: addr, info: byrefInfo, followForward: false, name: "object");
2501
2502 generator.emitDispose(CGF, field: addr);
2503 }
2504
2505 CGF.FinishFunction();
2506
2507 return Fn;
2508}
2509
2510/// Build the dispose helper for a __block variable.
2511static llvm::Constant *buildByrefDisposeHelper(CodeGenModule &CGM,
2512 const BlockByrefInfo &byrefInfo,
2513 BlockByrefHelpers &generator) {
2514 CodeGenFunction CGF(CGM);
2515 return generateByrefDisposeHelper(CGF, byrefInfo, generator);
2516}
2517
2518/// Lazily build the copy and dispose helpers for a __block variable
2519/// with the given information.
2520template <class T>
2521static T *buildByrefHelpers(CodeGenModule &CGM, const BlockByrefInfo &byrefInfo,
2522 T &&generator) {
2523 llvm::FoldingSetNodeID id;
2524 generator.Profile(id);
2525
2526 void *insertPos;
2527 BlockByrefHelpers *node
2528 = CGM.ByrefHelpersCache.FindNodeOrInsertPos(ID: id, InsertPos&: insertPos);
2529 if (node) return static_cast<T*>(node);
2530
2531 generator.CopyHelper = buildByrefCopyHelper(CGM, byrefInfo, generator);
2532 generator.DisposeHelper = buildByrefDisposeHelper(CGM, byrefInfo, generator);
2533
2534 T *copy = new (CGM.getContext()) T(std::forward<T>(generator));
2535 CGM.ByrefHelpersCache.InsertNode(copy, insertPos);
2536 return copy;
2537}
2538
2539/// Build the copy and dispose helpers for the given __block variable
2540/// emission. Places the helpers in the global cache. Returns null
2541/// if no helpers are required.
2542BlockByrefHelpers *
2543CodeGenFunction::buildByrefHelpers(llvm::StructType &byrefType,
2544 const AutoVarEmission &emission) {
2545 const VarDecl &var = *emission.Variable;
2546 assert(var.isEscapingByref() &&
2547 "only escaping __block variables need byref helpers");
2548
2549 QualType type = var.getType();
2550
2551 auto &byrefInfo = getBlockByrefInfo(var: &var);
2552
2553 // The alignment we care about for the purposes of uniquing byref
2554 // helpers is the alignment of the actual byref value field.
2555 CharUnits valueAlignment =
2556 byrefInfo.ByrefAlignment.alignmentAtOffset(offset: byrefInfo.FieldOffset);
2557
2558 if (const CXXRecordDecl *record = type->getAsCXXRecordDecl()) {
2559 const Expr *copyExpr =
2560 CGM.getContext().getBlockVarCopyInit(VD: &var).getCopyExpr();
2561 if (!copyExpr && record->hasTrivialDestructor()) return nullptr;
2562
2563 return ::buildByrefHelpers(
2564 CGM, byrefInfo, generator: CXXByrefHelpers(valueAlignment, type, copyExpr));
2565 }
2566 if (type.hasAddressDiscriminatedPointerAuth()) {
2567 return ::buildByrefHelpers(
2568 CGM, byrefInfo, generator: AddressDiscriminatedByrefHelpers(valueAlignment, type));
2569 }
2570 // If type is a non-trivial C struct type that is non-trivial to
2571 // destructly move or destroy, build the copy and dispose helpers.
2572 if (type.isNonTrivialToPrimitiveDestructiveMove() == QualType::PCK_Struct ||
2573 type.isDestructedType() == QualType::DK_nontrivial_c_struct)
2574 return ::buildByrefHelpers(
2575 CGM, byrefInfo, generator: NonTrivialCStructByrefHelpers(valueAlignment, type));
2576
2577 // Otherwise, if we don't have a retainable type, there's nothing to do.
2578 // that the runtime does extra copies.
2579 if (!type->isObjCRetainableType()) return nullptr;
2580
2581 Qualifiers qs = type.getQualifiers();
2582
2583 // If we have lifetime, that dominates.
2584 if (Qualifiers::ObjCLifetime lifetime = qs.getObjCLifetime()) {
2585 switch (lifetime) {
2586 case Qualifiers::OCL_None: llvm_unreachable("impossible");
2587
2588 // These are just bits as far as the runtime is concerned.
2589 case Qualifiers::OCL_ExplicitNone:
2590 case Qualifiers::OCL_Autoreleasing:
2591 return nullptr;
2592
2593 // Tell the runtime that this is ARC __weak, called by the
2594 // byref routines.
2595 case Qualifiers::OCL_Weak:
2596 return ::buildByrefHelpers(CGM, byrefInfo,
2597 generator: ARCWeakByrefHelpers(valueAlignment));
2598
2599 // ARC __strong __block variables need to be retained.
2600 case Qualifiers::OCL_Strong:
2601 // Block pointers need to be copied, and there's no direct
2602 // transfer possible.
2603 if (type->isBlockPointerType()) {
2604 return ::buildByrefHelpers(CGM, byrefInfo,
2605 generator: ARCStrongBlockByrefHelpers(valueAlignment));
2606
2607 // Otherwise, we transfer ownership of the retain from the stack
2608 // to the heap.
2609 } else {
2610 return ::buildByrefHelpers(CGM, byrefInfo,
2611 generator: ARCStrongByrefHelpers(valueAlignment));
2612 }
2613 }
2614 llvm_unreachable("fell out of lifetime switch!");
2615 }
2616
2617 BlockFieldFlags flags;
2618 if (type->isBlockPointerType()) {
2619 flags |= BLOCK_FIELD_IS_BLOCK;
2620 } else if (CGM.getContext().isObjCNSObjectType(Ty: type) ||
2621 type->isObjCObjectPointerType()) {
2622 flags |= BLOCK_FIELD_IS_OBJECT;
2623 } else {
2624 return nullptr;
2625 }
2626
2627 if (type.isObjCGCWeak())
2628 flags |= BLOCK_FIELD_IS_WEAK;
2629
2630 return ::buildByrefHelpers(CGM, byrefInfo,
2631 generator: ObjectByrefHelpers(valueAlignment, flags));
2632}
2633
2634Address CodeGenFunction::emitBlockByrefAddress(Address baseAddr,
2635 const VarDecl *var,
2636 bool followForward) {
2637 auto &info = getBlockByrefInfo(var);
2638 return emitBlockByrefAddress(baseAddr, info, followForward, name: var->getName());
2639}
2640
2641Address CodeGenFunction::emitBlockByrefAddress(Address baseAddr,
2642 const BlockByrefInfo &info,
2643 bool followForward,
2644 const llvm::Twine &name) {
2645 // Chase the forwarding address if requested.
2646 if (followForward) {
2647 Address forwardingAddr = Builder.CreateStructGEP(Addr: baseAddr, Index: 1, Name: "forwarding");
2648 baseAddr = Address(Builder.CreateLoad(Addr: forwardingAddr), info.Type,
2649 info.ByrefAlignment);
2650 }
2651
2652 return Builder.CreateStructGEP(Addr: baseAddr, Index: info.FieldIndex, Name: name);
2653}
2654
2655/// BuildByrefInfo - This routine changes a __block variable declared as T x
2656/// into:
2657///
2658/// struct {
2659/// void *__isa;
2660/// void *__forwarding;
2661/// int32_t __flags;
2662/// int32_t __size;
2663/// void *__copy_helper; // only if needed
2664/// void *__destroy_helper; // only if needed
2665/// void *__byref_variable_layout;// only if needed
2666/// char padding[X]; // only if needed
2667/// T x;
2668/// } x
2669///
2670const BlockByrefInfo &CodeGenFunction::getBlockByrefInfo(const VarDecl *D) {
2671 auto it = BlockByrefInfos.find(Val: D);
2672 if (it != BlockByrefInfos.end())
2673 return it->second;
2674
2675 QualType Ty = D->getType();
2676
2677 CharUnits size;
2678 SmallVector<llvm::Type *, 8> types;
2679
2680 // void *__isa;
2681 types.push_back(Elt: VoidPtrTy);
2682 size += getPointerSize();
2683
2684 // void *__forwarding;
2685 types.push_back(Elt: VoidPtrTy);
2686 size += getPointerSize();
2687
2688 // int32_t __flags;
2689 types.push_back(Elt: Int32Ty);
2690 size += CharUnits::fromQuantity(Quantity: 4);
2691
2692 // int32_t __size;
2693 types.push_back(Elt: Int32Ty);
2694 size += CharUnits::fromQuantity(Quantity: 4);
2695
2696 // Note that this must match *exactly* the logic in buildByrefHelpers.
2697 bool hasCopyAndDispose = getContext().BlockRequiresCopying(Ty, D);
2698 if (hasCopyAndDispose) {
2699 /// void *__copy_helper;
2700 types.push_back(Elt: VoidPtrTy);
2701 size += getPointerSize();
2702
2703 /// void *__destroy_helper;
2704 types.push_back(Elt: VoidPtrTy);
2705 size += getPointerSize();
2706 }
2707
2708 bool HasByrefExtendedLayout = false;
2709 Qualifiers::ObjCLifetime Lifetime = Qualifiers::OCL_None;
2710 if (getContext().getByrefLifetime(Ty, Lifetime, HasByrefExtendedLayout) &&
2711 HasByrefExtendedLayout) {
2712 /// void *__byref_variable_layout;
2713 types.push_back(Elt: VoidPtrTy);
2714 size += CharUnits::fromQuantity(Quantity: PointerSizeInBytes);
2715 }
2716
2717 // T x;
2718 llvm::Type *varTy = ConvertTypeForMem(T: Ty);
2719
2720 bool packed = false;
2721 CharUnits varAlign = getContext().getDeclAlign(D);
2722 CharUnits varOffset = size.alignTo(Align: varAlign);
2723
2724 // We may have to insert padding.
2725 if (varOffset != size) {
2726 llvm::Type *paddingTy =
2727 llvm::ArrayType::get(ElementType: Int8Ty, NumElements: (varOffset - size).getQuantity());
2728
2729 types.push_back(Elt: paddingTy);
2730 size = varOffset;
2731
2732 // Conversely, we might have to prevent LLVM from inserting padding.
2733 } else if (CGM.getDataLayout().getABITypeAlign(Ty: varTy) >
2734 uint64_t(varAlign.getQuantity())) {
2735 packed = true;
2736 }
2737 types.push_back(Elt: varTy);
2738
2739 llvm::StructType *byrefType = llvm::StructType::create(
2740 Context&: getLLVMContext(), Elements: types, Name: "struct.__block_byref_" + D->getNameAsString(),
2741 isPacked: packed);
2742
2743 BlockByrefInfo info;
2744 info.Type = byrefType;
2745 info.FieldIndex = types.size() - 1;
2746 info.FieldOffset = varOffset;
2747 info.ByrefAlignment = std::max(a: varAlign, b: getPointerAlign());
2748
2749 auto pair = BlockByrefInfos.insert(KV: {D, info});
2750 assert(pair.second && "info was inserted recursively?");
2751 return pair.first->second;
2752}
2753
2754/// Initialize the structural components of a __block variable, i.e.
2755/// everything but the actual object.
2756void CodeGenFunction::emitByrefStructureInit(const AutoVarEmission &emission) {
2757 // Find the address of the local.
2758 Address addr = emission.Addr;
2759
2760 // That's an alloca of the byref structure type.
2761 llvm::StructType *byrefType = cast<llvm::StructType>(Val: addr.getElementType());
2762
2763 unsigned nextHeaderIndex = 0;
2764 CharUnits nextHeaderOffset;
2765 auto storeHeaderField = [&](llvm::Value *value, CharUnits fieldSize,
2766 const Twine &name, bool isFunction = false) {
2767 auto fieldAddr = Builder.CreateStructGEP(Addr: addr, Index: nextHeaderIndex, Name: name);
2768 if (isFunction) {
2769 if (auto &Schema = CGM.getCodeGenOpts()
2770 .PointerAuth.BlockByrefHelperFunctionPointers) {
2771 auto PointerAuth = EmitPointerAuthInfo(
2772 Schema, StorageAddress: fieldAddr.emitRawPointer(CGF&: *this), SchemaDecl: GlobalDecl(), SchemaType: QualType());
2773 value = EmitPointerAuthSign(Info: PointerAuth, Pointer: value);
2774 }
2775 }
2776 Builder.CreateStore(Val: value, Addr: fieldAddr);
2777
2778 nextHeaderIndex++;
2779 nextHeaderOffset += fieldSize;
2780 };
2781
2782 // Build the byref helpers if necessary. This is null if we don't need any.
2783 BlockByrefHelpers *helpers = buildByrefHelpers(byrefType&: *byrefType, emission);
2784
2785 const VarDecl &D = *emission.Variable;
2786 QualType type = D.getType();
2787
2788 bool HasByrefExtendedLayout = false;
2789 Qualifiers::ObjCLifetime ByrefLifetime = Qualifiers::OCL_None;
2790 bool ByRefHasLifetime =
2791 getContext().getByrefLifetime(Ty: type, Lifetime&: ByrefLifetime, HasByrefExtendedLayout);
2792
2793 llvm::Value *V;
2794
2795 // Initialize the 'isa', which is just 0 or 1.
2796 int isa = 0;
2797 if (type.isObjCGCWeak())
2798 isa = 1;
2799 V = Builder.CreateIntToPtr(V: Builder.getInt32(C: isa), DestTy: Int8PtrTy, Name: "isa");
2800 storeHeaderField(V, getPointerSize(), "byref.isa");
2801
2802 // Store the address of the variable into its own forwarding pointer.
2803 storeHeaderField(addr.emitRawPointer(CGF&: *this), getPointerSize(),
2804 "byref.forwarding");
2805
2806 // Blocks ABI:
2807 // c) the flags field is set to either 0 if no helper functions are
2808 // needed or BLOCK_BYREF_HAS_COPY_DISPOSE if they are,
2809 BlockFlags flags;
2810 if (helpers) flags |= BLOCK_BYREF_HAS_COPY_DISPOSE;
2811 if (ByRefHasLifetime) {
2812 if (HasByrefExtendedLayout) flags |= BLOCK_BYREF_LAYOUT_EXTENDED;
2813 else switch (ByrefLifetime) {
2814 case Qualifiers::OCL_Strong:
2815 flags |= BLOCK_BYREF_LAYOUT_STRONG;
2816 break;
2817 case Qualifiers::OCL_Weak:
2818 flags |= BLOCK_BYREF_LAYOUT_WEAK;
2819 break;
2820 case Qualifiers::OCL_ExplicitNone:
2821 flags |= BLOCK_BYREF_LAYOUT_UNRETAINED;
2822 break;
2823 case Qualifiers::OCL_None:
2824 if (!type->isObjCObjectPointerType() && !type->isBlockPointerType())
2825 flags |= BLOCK_BYREF_LAYOUT_NON_OBJECT;
2826 break;
2827 default:
2828 break;
2829 }
2830 if (CGM.getLangOpts().ObjCGCBitmapPrint) {
2831 printf(format: "\n Inline flag for BYREF variable layout (%d):", flags.getBitMask());
2832 if (flags & BLOCK_BYREF_HAS_COPY_DISPOSE)
2833 printf(format: " BLOCK_BYREF_HAS_COPY_DISPOSE");
2834 if (flags & BLOCK_BYREF_LAYOUT_MASK) {
2835 BlockFlags ThisFlag(flags.getBitMask() & BLOCK_BYREF_LAYOUT_MASK);
2836 if (ThisFlag == BLOCK_BYREF_LAYOUT_EXTENDED)
2837 printf(format: " BLOCK_BYREF_LAYOUT_EXTENDED");
2838 if (ThisFlag == BLOCK_BYREF_LAYOUT_STRONG)
2839 printf(format: " BLOCK_BYREF_LAYOUT_STRONG");
2840 if (ThisFlag == BLOCK_BYREF_LAYOUT_WEAK)
2841 printf(format: " BLOCK_BYREF_LAYOUT_WEAK");
2842 if (ThisFlag == BLOCK_BYREF_LAYOUT_UNRETAINED)
2843 printf(format: " BLOCK_BYREF_LAYOUT_UNRETAINED");
2844 if (ThisFlag == BLOCK_BYREF_LAYOUT_NON_OBJECT)
2845 printf(format: " BLOCK_BYREF_LAYOUT_NON_OBJECT");
2846 }
2847 printf(format: "\n");
2848 }
2849 }
2850 storeHeaderField(llvm::ConstantInt::get(Ty: IntTy, V: flags.getBitMask()),
2851 getIntSize(), "byref.flags");
2852
2853 CharUnits byrefSize = CGM.GetTargetTypeStoreSize(Ty: byrefType);
2854 V = llvm::ConstantInt::get(Ty: IntTy, V: byrefSize.getQuantity());
2855 storeHeaderField(V, getIntSize(), "byref.size");
2856
2857 if (helpers) {
2858 storeHeaderField(helpers->CopyHelper, getPointerSize(), "byref.copyHelper",
2859 /*isFunction=*/true);
2860 storeHeaderField(helpers->DisposeHelper, getPointerSize(),
2861 "byref.disposeHelper", /*isFunction=*/true);
2862 }
2863
2864 if (ByRefHasLifetime && HasByrefExtendedLayout) {
2865 auto layoutInfo = CGM.getObjCRuntime().BuildByrefLayout(CGM, T: type);
2866 storeHeaderField(layoutInfo, getPointerSize(), "byref.layout");
2867 }
2868}
2869
2870void CodeGenFunction::BuildBlockRelease(llvm::Value *V, BlockFieldFlags flags,
2871 bool CanThrow) {
2872 llvm::FunctionCallee F = CGM.getBlockObjectDispose();
2873 llvm::Value *args[] = {V,
2874 llvm::ConstantInt::get(Ty: Int32Ty, V: flags.getBitMask())};
2875
2876 if (CanThrow)
2877 EmitRuntimeCallOrInvoke(callee: F, args);
2878 else
2879 EmitNounwindRuntimeCall(callee: F, args);
2880}
2881
2882void CodeGenFunction::enterByrefCleanup(CleanupKind Kind, Address Addr,
2883 BlockFieldFlags Flags,
2884 bool LoadBlockVarAddr, bool CanThrow) {
2885 EHStack.pushCleanup<CallBlockRelease>(Kind, A: Addr, A: Flags, A: LoadBlockVarAddr,
2886 A: CanThrow);
2887}
2888
2889/// Adjust the declaration of something from the blocks API.
2890static void configureBlocksRuntimeObject(CodeGenModule &CGM,
2891 llvm::Constant *C) {
2892 auto *GV = cast<llvm::GlobalValue>(Val: C->stripPointerCasts());
2893
2894 if (!CGM.getCodeGenOpts().StaticClosure &&
2895 CGM.getTarget().getTriple().isOSBinFormatCOFF()) {
2896 const IdentifierInfo &II = CGM.getContext().Idents.get(Name: C->getName());
2897 TranslationUnitDecl *TUDecl = CGM.getContext().getTranslationUnitDecl();
2898 DeclContext *DC = TranslationUnitDecl::castToDeclContext(D: TUDecl);
2899
2900 assert((isa<llvm::Function>(C->stripPointerCasts()) ||
2901 isa<llvm::GlobalVariable>(C->stripPointerCasts())) &&
2902 "expected Function or GlobalVariable");
2903
2904 const NamedDecl *ND = nullptr;
2905 for (const auto *Result : DC->lookup(Name: &II))
2906 if ((ND = dyn_cast<FunctionDecl>(Val: Result)) ||
2907 (ND = dyn_cast<VarDecl>(Val: Result)))
2908 break;
2909
2910 if (GV->isDeclaration() && (!ND || !ND->hasAttr<DLLExportAttr>())) {
2911 GV->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass);
2912 GV->setLinkage(llvm::GlobalValue::ExternalLinkage);
2913 } else {
2914 GV->setDLLStorageClass(llvm::GlobalValue::DLLExportStorageClass);
2915 GV->setLinkage(llvm::GlobalValue::ExternalLinkage);
2916 }
2917 }
2918
2919 if (CGM.getLangOpts().BlocksRuntimeOptional && GV->isDeclaration() &&
2920 GV->hasExternalLinkage())
2921 GV->setLinkage(llvm::GlobalValue::ExternalWeakLinkage);
2922
2923 CGM.setDSOLocal(GV);
2924}
2925
2926llvm::FunctionCallee CodeGenModule::getBlockObjectDispose() {
2927 if (BlockObjectDispose)
2928 return BlockObjectDispose;
2929
2930 QualType args[] = {Context.VoidPtrTy, Context.IntTy};
2931 BlockObjectDispose =
2932 CreateRuntimeFunction(ReturnTy: Context.VoidTy, ArgTys: args, Name: "_Block_object_dispose");
2933 configureBlocksRuntimeObject(
2934 CGM&: *this, C: cast<llvm::Constant>(Val: BlockObjectDispose.getCallee()));
2935 return BlockObjectDispose;
2936}
2937
2938llvm::FunctionCallee CodeGenModule::getBlockObjectAssign() {
2939 if (BlockObjectAssign)
2940 return BlockObjectAssign;
2941
2942 QualType args[] = {Context.VoidPtrTy, Context.VoidPtrTy, Context.IntTy};
2943 BlockObjectAssign =
2944 CreateRuntimeFunction(ReturnTy: Context.VoidTy, ArgTys: args, Name: "_Block_object_assign");
2945 configureBlocksRuntimeObject(
2946 CGM&: *this, C: cast<llvm::Constant>(Val: BlockObjectAssign.getCallee()));
2947 return BlockObjectAssign;
2948}
2949
2950llvm::Constant *CodeGenModule::getNSConcreteGlobalBlock() {
2951 if (NSConcreteGlobalBlock)
2952 return NSConcreteGlobalBlock;
2953
2954 NSConcreteGlobalBlock = GetOrCreateLLVMGlobal(
2955 MangledName: "_NSConcreteGlobalBlock", Ty: Int8PtrTy, AddrSpace: LangAS::Default, D: nullptr);
2956 configureBlocksRuntimeObject(CGM&: *this, C: NSConcreteGlobalBlock);
2957 return NSConcreteGlobalBlock;
2958}
2959
2960llvm::Constant *CodeGenModule::getNSConcreteStackBlock() {
2961 if (NSConcreteStackBlock)
2962 return NSConcreteStackBlock;
2963
2964 NSConcreteStackBlock = GetOrCreateLLVMGlobal(
2965 MangledName: "_NSConcreteStackBlock", Ty: Int8PtrTy, AddrSpace: LangAS::Default, D: nullptr);
2966 configureBlocksRuntimeObject(CGM&: *this, C: NSConcreteStackBlock);
2967 return NSConcreteStackBlock;
2968}
2969